R1c: catalog readiness no longer compiles every kernel

.gitattributes

@@ -0,0 +1,7 @@
+# Tracked files are checked out as their committed bytes on every platform.
+# Benchmark corpora, reference-export sidecars and documentation audits are
+# compared by sha256, which Windows end-of-line conversion would change.
+* -text
+# Windows command scripts keep the platform's line endings.
+*.cmd text
+*.bat text

.github/r1-debug/cold_prep.py

@@ -0,0 +1,104 @@
+"""TEMPORARY (surface R1): time cold function-catalog preparation phase by phase.
+
+Wall time per status row, CPU time of this process and its finished children,
+and (R1_NUMBA_EVENTS=1) the self time of every Numba compilation.
+"""
+
+import collections
+import json
+import os
+import platform
+import sys
+import time
+
+START = time.perf_counter()
+rows = []
+
+
+def cpu_seconds():
+    t = os.times()
+    return t.user + t.system + t.children_user + t.children_system
+
+
+def status(message):
+    rows.append((time.perf_counter() - START, cpu_seconds(), str(message)))
+
+
+def numba_compile_report(listener):
+    stack = []
+    self_time = collections.Counter()
+    count = collections.Counter()
+    for timestamp, event in listener.buffer:
+        function = event.data["dispatcher"].py_func
+        name = f"{getattr(function, '__module__', '?')}.{getattr(function, '__qualname__', '?')}"
+        if event.is_start:
+            stack.append([name, timestamp, 0.0])
+        elif stack:
+            started_name, started, child = stack.pop()
+            total = timestamp - started
+            self_time[started_name] += total - child
+            count[started_name] += 1
+            if stack:
+                stack[-1][2] += total
+    total = sum(self_time.values())
+    print(f"NUMBA compile self time {total:.1f}s over {sum(count.values())} compilations of {len(count)} functions")
+    by_module = collections.Counter()
+    for name, seconds in self_time.items():
+        by_module[name.split(".<locals>")[0].rsplit(".", 1)[0]] += seconds
+    print("-- by module")
+    for module, seconds in by_module.most_common(25):
+        print(f"{seconds:8.2f}s  {module}")
+    print("-- by function (signatures compiled)")
+    for name, seconds in self_time.most_common(40):
+        print(f"{seconds:8.2f}s  x{count[name]:<3d} {name}")
+
+
+def main():
+    out = sys.argv[1]
+    print("platform", platform.platform(), platform.machine(), "cpus", os.cpu_count())
+    print("NUMBA_CACHE_DIR", os.environ.get("NUMBA_CACHE_DIR"), "OPENHCS_CPU_ONLY", os.environ.get("OPENHCS_CPU_ONLY"))
+    status("process started")
+    from openhcs.processing.backends.lib_registry.registry_service import RegistryService
+
+    status("registry service imported")
+    if os.environ.get("R1_NO_CHILDREN"):
+        from openhcs.core.processing_preparation import PreparationCacheBatch
+
+        PreparationCacheBatch.populate_child_caches = lambda *args, **kwargs: None
+    listener = None
+    if os.environ.get("R1_NUMBA_EVENTS"):
+        from numba.core import event
+
+        listener = event.RecordingListener()
+        event.register("numba:compile", listener)
+    if os.environ.get("R1_PROFILE"):
+        import cProfile
+        import pstats
+
+        profile = cProfile.Profile()
+        profile.runcall(RegistryService.prepare_in_current_process, status_callback=status)
+        pstats.Stats(profile).sort_stats("cumulative").print_stats(70)
+    else:
+        RegistryService.prepare_in_current_process(status_callback=status)
+    status("prepared")
+    for name in filter(None, os.environ.get("R1_FIRST_USE", "").split(",")):
+        from openhcs.core.callable_contract import CallableContract
+
+        used = next(m.func for m in RegistryService._metadata_cache.values() if m.func.__name__ == name)
+        CallableContract.from_prepared_callable(used)
+        status(f"first use prepared {name}")
+    with open(out, "w", encoding="utf-8") as handle:
+        for wall, cpu, message in rows:
+            handle.write(json.dumps({"t": wall, "cpu": cpu, "message": message}) + "\n")
+    gaps = sorted(
+        ((b[0] - a[0], b[1] - a[1], a[2], b[2]) for a, b in zip(rows, rows[1:])), reverse=True
+    )
+    print(f"TOTAL wall {rows[-1][0]:.1f}s cpu {rows[-1][1]:.1f}s over {len(rows)} status rows")
+    for wall, cpu, before, after in gaps[:30]:
+        print(f"wall {wall:7.2f}s cpu {cpu:7.2f}s  {before[:100]}  ->  {after[:100]}")
+    if listener is not None:
+        numba_compile_report(listener)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/r1-debug.yml

@@ -0,0 +1,81 @@
+# TEMPORARY (surface R1). Runs one failing piece on one failing runner.
+# Deleted before the PR is final.
+#
+# Dispatch:  gh workflow run r1-debug.yml --ref <branch> -f target=parity-macos
+# GitHub only dispatches workflows that exist on the default branch, so a push
+# to an R1 branch whose head commit message contains "[r1 <target>]" runs the
+# same job. A push without a tag starts nothing.
+name: R1 debug
+
+on:
+  push:
+    branches: ["refactor/R1c-cold-start"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: r1-debug-${{ github.ref }}-${{ github.event.inputs.target || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  cold-intel:
+    name: cold catalog preparation (macos-15-intel)
+    if: contains(github.event.head_commit.message, '[r1 cold-intel]')
+    runs-on: macos-15-intel
+    timeout-minutes: 45
+    defaults:
+      run:
+        shell: bash
+    env:
+      OPENHCS_CPU_ONLY: true
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Build and install the desktop candidate wheels
+        run: |
+          python -m pip install --upgrade pip build
+          desktop_extras=$(python "$GITHUB_WORKSPACE/scripts/sync_mcp_release_metadata.py" --print-desktop-extras)
+          python -m scripts.install_ci_candidate \
+            --dependency-source submodules \
+            --wheel-directory "$RUNNER_TEMP/openhcs-desktop-candidate-wheels" \
+            --extras "$desktop_extras"
+
+      - name: A. Cold preparation as the installed GUI smoke starts it (kernels compile in the server)
+        run: |
+          mkdir -p "$RUNNER_TEMP/r1" && cd "$RUNNER_TEMP/r1"
+          sysctl -n machdep.cpu.brand_string hw.ncpu || true
+          R1_FIRST_USE=create_composite,threshold,identify_primary_objects,watershed_cellprofiler4 R1_NUMBA_EVENTS=1 XDG_CACHE_HOME="$RUNNER_TEMP/r1/a-cache" XDG_CONFIG_HOME="$RUNNER_TEMP/r1/a-config" XDG_DATA_HOME="$RUNNER_TEMP/r1/a-data" \
+            python -I "$GITHUB_WORKSPACE/.github/r1-debug/cold_prep.py" "$RUNNER_TEMP/r1/a-cold-serial.jsonl"
+
+      - name: C. Second start on warm caches
+        run: |
+          cd "$RUNNER_TEMP/r1"
+          XDG_CACHE_HOME="$RUNNER_TEMP/r1/a-cache" XDG_CONFIG_HOME="$RUNNER_TEMP/r1/a-config" XDG_DATA_HOME="$RUNNER_TEMP/r1/a-data" \
+            python -I "$GITHUB_WORKSPACE/.github/r1-debug/cold_prep.py" "$RUNNER_TEMP/r1/c-warm.jsonl"
+
+      - name: D. Profile of catalog discovery (cold function catalog, warm kernels)
+        run: |
+          cd "$RUNNER_TEMP/r1"
+          R1_PROFILE=1 XDG_CACHE_HOME="$RUNNER_TEMP/r1/d-cache" XDG_CONFIG_HOME="$RUNNER_TEMP/r1/d-config" XDG_DATA_HOME="$RUNNER_TEMP/r1/d-data" \
+            python -I "$GITHUB_WORKSPACE/.github/r1-debug/cold_prep.py" "$RUNNER_TEMP/r1/d-profile.jsonl" | head -170
+
+      - name: E. Import time of the registry service
+        run: |
+          cd "$RUNNER_TEMP/r1"
+          python -I -X importtime -c "from openhcs.processing.backends.lib_registry.registry_service import RegistryService" 2> importtime.txt
+          sort -t'|' -k2 -n -r importtime.txt | head -40
+
+      - name: Upload timelines
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: r1-cold-intel
+          path: ${{ runner.temp }}/r1/*.jsonl
+          retention-days: 7

benchmark/adapters/openhcs.py

@@ -18,6 +18,8 @@
 from pathlib import Path
 from typing import Any
 
+from objectstate import replace_raw
+
 from benchmark.adapters.cppipe_source import (
     CPPipeSourceRequest,
     CPPipeSourceResolution,
@@ -308,7 +310,6 @@ def _execution_global_config(
         request: OpenHCSRunRequest,
         *,
         output_suffix: str,
-        output_dataset_root: Path,
         compilation_debug_config: CompilationDebugConfig,
     ) -> GlobalPipelineConfig:
         """Select benchmark output paths without replacing declared path policy."""
@@ -330,7 +331,6 @@ def _execution_global_config(
             ),
             compilation_debug_config=compilation_debug_config,
             materialize_runtime_artifacts=request.materialize_runtime_artifacts,
-            materialization_results_path=output_dataset_root / "results",
         )
 
     def _run_converted_cppipe_pipeline(
@@ -361,9 +361,6 @@ def _run_converted_cppipe_pipeline(
             ) from exc
 
         output_suffix = f"_{request.pipeline_name}_converted_cppipe"
-        output_dataset_root = (
-            request.output_dir / f"{request.dataset_path.name}{output_suffix}"
-        )
         generated_source_path = request.output_dir / f"{cppipe_path.stem}_openhcs.py"
         source_workspace_path = (
             request.output_dir
@@ -417,7 +414,9 @@ def _run_converted_cppipe_pipeline(
                 "UI/ZMQ FunctionStep source before execution."
             )
         if compilation_debug_config.enabled:
-            pipeline_config = replace(
+            # replace_raw keeps the imported config's unset fields inheriting
+            # from the global config; dataclasses.replace would freeze them.
+            pipeline_config = replace_raw(
                 pipeline_config,
                 compilation_debug_config=LazyCompilationDebugConfig(
                     enabled=compilation_debug_config.enabled,
@@ -430,7 +429,6 @@ def _run_converted_cppipe_pipeline(
         global_config = self._execution_global_config(
             request,
             output_suffix=output_suffix,
-            output_dataset_root=output_dataset_root,
             compilation_debug_config=compilation_debug_config,
         )
         ensure_global_config_context(GlobalPipelineConfig, global_config)
@@ -550,7 +548,6 @@ def _run_converted_cppipe_pipeline(
                     raise ToolExecutionError(equivalence_failure_message)
 
         metric_results = self._metric_results(request.metrics)
-        output_dataset_root.mkdir(parents=True, exist_ok=True)
         execution_output_root.mkdir(parents=True, exist_ok=True)
 
         provenance = {

benchmark/cellprofiler_benchmark_cli.py

@@ -6,6 +6,7 @@
 import json
 import math
 import os
+import signal
 import sys
 from abc import ABC, abstractmethod
 from collections.abc import Sequence
@@ -1069,6 +1070,10 @@ def create_benchmark_argument_parser() -> argparse.ArgumentParser:
 def main(argv: Sequence[str] | None = None) -> int:
     """Run the registered benchmark CLI."""
 
+    if sys.platform == "win32":
+        # Ctrl+Break is the Windows console interrupt a launcher can deliver
+        # to this process alone; it stops a run exactly as Ctrl+C does.
+        signal.signal(signal.SIGBREAK, signal.default_int_handler)
     arguments = tuple(sys.argv[1:] if argv is None else argv)
     parser = create_benchmark_argument_parser()
     parsed = parser.parse_args(arguments)

benchmark/openhcs_measured_run.py

@@ -8,10 +8,7 @@
 import sys
 import time
 from collections.abc import Callable, Mapping
-from dataclasses import (
-    dataclass,
-    field,
-)
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
@@ -134,7 +131,12 @@ class _ZMQProgressTimingObserver:
     compile_completed_at: float | None = None
     execution_started_at: float | None = None
     execution_completed_at: float | None = None
-    last_progress_monotonic: float = field(default_factory=time.monotonic)
+    last_progress_monotonic: float | None = None
+    """Monotonic time of the latest progress; None until a run is submitted.
+
+    Starting the execution server is not execution: its client owns the
+    startup readiness timeout, so silence is measured from begin_run.
+    """
     last_progress_phase: str = ""
     last_progress_status: str = ""
     on_event: Callable[[Mapping[str, Any]], None] | None = None
@@ -217,6 +219,8 @@ def _duration(started_at: float | None, ended_at: float | None) -> float | None:
 
     def inactivity_seconds(self, *, observed_at: float | None = None) -> float:
         """Return elapsed monotonic time since the latest server progress event."""
+        if self.last_progress_monotonic is None:
+            return 0.0
         current = time.monotonic() if observed_at is None else observed_at
         return max(0.0, current - self.last_progress_monotonic)
 

docs/refactor/consolidation-20261009/B6d-remaining-fixes.md

@@ -0,0 +1,37 @@
+# B6d: the fixes from B6 that main still needs
+
+B6 (#1195) repaired main's CI in one branch that also reformatted 412 files and re-pinned the documentation audits. #1212 took its E1 boundary fix and #1214 its trigger restructuring. This is the rest: each fix below changes behaviour. Formatting commits, the ratchet mixin moves, the documentation re-pins, the workflow edits and the pyqt-reactive pin are not carried.
+
+## Fixes
+
+Tests named here fail on main `ae6ddace5` and pass on this branch unless marked otherwise.
+
+| Fix | Cause | Evidence |
+|---|---|---|
+| Requestless UI-bridge operations pass the connection by keyword | the connection landed in the request slot | `test_capabilities.py::test_requestless_ui_bridge_capability_passes_connection_as_connection` |
+| A batch stopped while compiling ends cancelled | the stop surfaced as an execution failure | `test_execution_control_stop_during_compile.py` |
+| Metadata document targets follow the backend's declaration | a virtual backend (OMERO) got a local metadata document | `test_function_outputs.py::test_metadata_document_targets_follow_backend_declaration` |
+| Catalog leaves out modules whose distributions the package markers exclude | Intel macOS excludes `openhcs-basicpy` but the catalog imported its module | `test_library_registry_discovery.py` (7 cases) |
+| Cache preparation uses `os.cpu_count()` where there is no affinity API | macOS prepared the kernel cache serially | `test_preparation_admission.py::test_platform_without_affinity_admits_its_online_cpus` |
+| Scalar components are reduced per pixel group | contributors were pooled across runtime planes, dropping `site` | `test_aggregate_source_scope.py::test_axis_varying_between_runtime_planes_is_kept_not_reduced` |
+| The one-plane label projection belongs to every plane label mode | it lived only on `MatchImageStackLabels` | `test_singleton_rgb_label_invocation_477.py` (`SliceAlignedLabels` case) |
+| Benchmark adapter keeps the imported config's inheritance and declared output policy | `dataclasses.replace` froze unset fields; a results path was forced | `test_openhcs_adapter.py::test_openhcs_adapter_preserves_declared_main_flow_output_policy` |
+| Packaged knowledge recipes are placed by case name within a path budget | installed paths exceeded Windows MAX_PATH | `test_build_mcp_knowledge_assets.py` (does not import on main) |
+| GUI smoke waits for the window's endpoint start, reports the status name, and gives the bridge connection its transport config | the catalog was read before the endpoint existed; the result held a class; U4 moved the connection spec to formbridge, where `transport_endpoint` takes the config | `test_smoke_installed_gui.py` (does not import on main); `smoke_local.sh` GUI step |
+| Unit session creates its `QApplication` first; sibling test modules import through `tests` | an MCP transport created a `QCoreApplication`; bare imports fail under importlib mode | `test_ci_import_guards.py` guards the imports; it passes on main once the test files are fixed, so no production test fails on main |
+| STEP_INPUT filters match stored source paths | they matched virtual workspace names | no touched local test fails on main; B6 found it in the backend-microscope CI jobs |
+| Passthrough outputs keep their producer's filename qualifier | sibling surfaces merged into one file | same |
+| BaSiC reads metadata through `ImagePayload.of` | `.metadata` was read off plain arrays | not shown failing on main locally; B6 found it in CI |
+| Dataset rows carry the finished execution's error; demo reads `ViewerType.value` | the installed demo failed without a reason | installed demo job only |
+| Measured-run silence is counted from `begin_run` | server startup time counted as execution silence | not isolated by a local test |
+| `mahotas` is a core dependency | core catalog modules import it; it sat in an extra | packaging; python-314 job |
+| MCP smoke sends `partition_values` | G2 renamed `wells` | wheel-integration job only |
+| Windows: granularity accepts `'L'` uint32; custom-function sources are written as bytes; Ctrl+Break stops the CLI; dataset writability is probed by creating a file; `.gitattributes` keeps committed bytes; test process reaping skips process groups | each is a Windows-only difference | Windows jobs only; the POSIX paths are run by the touched tests |
+
+## Ratchet
+
+`OpenHCSMainWindow` and `PatternGroupExecutionRequest` are at their size budget. The window's four theme methods had no caller and are deleted (**unused**; guarded in `test_dead_ui_code_guards.py`). The passthrough helpers sit on `PatternGroupExecutionScope`.
+
+## Done when
+
+Touched tests pass, `smoke_local.sh` prints `smoke rc=0` with the GUI step, `guardrails_local.sh` ends `guardrails rc=0`, and the 30-workflow parity check passes.