ci: V-model pipeline for native-ci and GitHub CI (Refs #254)

.config/nextest.toml

@@ -0,0 +1,28 @@
+# cargo-nextest configuration (Refs #254).
+#
+# `default` is what developers get locally. `ci` is selected in the workflows
+# with `--profile ci`: no fail-fast so one red target does not hide the rest,
+# JUnit output for the runner to archive, and bounded retries only for tests
+# that are known to depend on a live terraphim_server process.
+
+[profile.default]
+retries = 0
+fail-fast = true
+slow-timeout = { period = "60s", terminate-after = 3 }
+
+[profile.ci]
+retries = 0
+fail-fast = false
+slow-timeout = { period = "60s", terminate-after = 3 }
+
+[profile.ci.junit]
+path = "junit.xml"
+
+# The integration binaries that shell out to a real terraphim_server
+# (TERRAPHIM_SERVER_BIN) are the only ones allowed a retry and a longer
+# slow-timeout, and only in CI: they wait on a live HTTP service. Everything
+# else must be deterministic; a flaky test is a bug, not a retry candidate.
+[[profile.ci.overrides]]
+filter = 'binary(/^(server_mode_tests|cross_mode_consistency_test|integration_tests|kg_ranking_integration_test)$/)'
+retries = 1
+slow-timeout = { period = "120s", terminate-after = 3 }

.gitea/workflows/native-ci.yml

@@ -1,25 +1,45 @@
 name: native-ci
+# V-model gate order (Refs #254): fast checks -> tests -> coverage/security ->
+# nightly UB gates. Kept in sync with .github/workflows/ci.yml; see
+# docs/ci-gap-analysis.md "Sync rule". Every step's first token must be on the
+# terraphim-gitea-runner allowlist (cargo, rustup, bash, test, ...); curl and
+# python are denied, so tools are provisioned with `cargo install`.
 on:
   push:
   workflow_dispatch:
 jobs:
-  build:
+  # Stage 1: fast checks (fmt, clippy, deny). Fails in seconds, not minutes.
+  check:
     runs-on: terraphim-native
     steps:
+      - run: rustup show active-toolchain
       - run: cargo fmt --all -- --check
       - run: cargo clippy --workspace --all-targets -- -D warnings
+      # #2171: enrichment feature is not covered by default-features clippy.
+      - run: cargo clippy -p terraphim_sessions --features enrichment -- -D warnings
+      # Supply chain: advisories, licences, bans, sources per deny.toml.
+      # cargo-deny is present on runner-5 only, so install idempotently.
+      - run: cargo install cargo-deny --locked --version 0.20.2
+      - run: cargo deny check
+  # Stage 2: build + full test gate against a real terraphim_server.
+  test:
+    needs: check
+    runs-on: terraphim-native
+    steps:
+      # #106: the terraphim_update signed-archive tests shell out to the
+      # `zipsign` binary on the host. It is installed at /usr/local/bin/zipsign
+      # on bigbox (see "Host Tooling" in gitea-infrastructure HANDOVER.md);
+      # runners carry their own CARGO_HOME and do not all have ~/.cargo/bin on
+      # PATH, so a user-local install is invisible to them. Fail fast with a
+      # pointer to the fix rather than 21 failing test targets.
+      - name: Check host tooling (zipsign)
+        # Note: the terraphim-gitea-runner command policy inspects the
+        # literal first token; `if`/`then`/`fi` shell keywords get rejected.
+        # Use `test` (the only conditional primitive on the allowlist) and
+        # `||` chaining instead. Refs #106.
+        run: |
+          test -x /usr/local/bin/zipsign && /usr/local/bin/zipsign --version || { echo "::error::zipsign not found on PATH. Install on the runner host: sudo install -m 0755 ~/.cargo/bin/zipsign /usr/local/bin/zipsign (see gitea-infrastructure HANDOVER.md, 'Host Tooling'). Refs #106"; exit 1; }
       - run: cargo build --workspace
-      # #84: broaden the workspace gate so integration tests, binary
-      # smoke tests, and example doctests participate in the gate. The
-      # narrower `--lib` gate silently skipped crates/terraphim_mcp_server
-      # tests/test_tools_list.rs and tests/test_all_mcp_tools.rs, which
-      # exercised real stdio/JSON-RPC round-trips against the
-      # terraphim_mcp_server binary. `--tests --bins --examples` adds
-      # those targets; `--lib` is kept so the existing crate-internal
-      # coverage is still exercised. `--no-fail-fast` makes all targets
-      # run even when one fails, so a single broken test does not hide
-      # the rest of the failures behind an early abort.
-      - run: cargo test --workspace --tests --bins --examples --lib --no-fail-fast
       # #113: build terraphim_server from terraphim-ai so the
       # server-binary-dependent integration tests have a real binary.
       # terraphim_server is not a workspace member here -- it lives in
@@ -41,22 +61,44 @@ jobs:
       # requirements (1.20.2) match both 1.20.2 and 1.21.0 in the registry
       # and cargo aborts with "patch resolved to more than one candidate".
       - run: cargo install --locked --git https://git.terraphim.cloud/terraphim/terraphim-ai --tag v1.21.3 --root /tmp/terraphim_server_install --config 'registries.terraphim.index="sparse+https://git.terraphim.cloud/api/packages/terraphim/cargo/"' --config 'registry.global-credential-providers=["cargo:token"]' --bin terraphim_server terraphim_server
-      # #113: run the integration tests that require a real
-      # terraphim_server binary. ensure_server_binary() (in
-      # cross_mode_consistency_test.rs / kg_ranking_integration_test.rs)
-      # and server_binary_path() (in integration_tests.rs) both resolve
-      # TERRAPHIM_SERVER_BIN first, so pointing the env var at the
-      # install root is enough.
-      - run: TERRAPHIM_SERVER_BIN=/tmp/terraphim_server_install/bin/terraphim_server cargo test -p terraphim_agent --test cross_mode_consistency_test -- --nocapture
-      - run: TERRAPHIM_SERVER_BIN=/tmp/terraphim_server_install/bin/terraphim_server cargo test -p terraphim_agent --test integration_tests -- --nocapture
-      - run: TERRAPHIM_SERVER_BIN=/tmp/terraphim_server_install/bin/terraphim_server cargo test -p terraphim_agent --test kg_ranking_integration_test -- --nocapture
-      # #2171: enrichment feature clippy + test invocations.
-      - run: cargo clippy -p terraphim_sessions --features enrichment -- -D warnings
-      - run: cargo test -p terraphim_sessions --features enrichment --lib --no-fail-fast
-      # #95: isolated packaged install-graph regression.
-      - run: cargo test -p terraphim_agent --test packaged_install_graph_regression -- --nocapture
-      # #118: repo guards -- duplicate-crate detection and the publish gate's own
-      # tests. Rust tests, not shell steps: the runner allowlist rejects any
-      # program that is not cargo ("policy rejected command: ... not on the
-      # allowlist"), which is what took CI down from #112 until now.
-      - run: cargo test -p terraphim_agent --test ci_guards -- --nocapture
+      # nextest: one binary per target, parallel scheduling, JUnit output
+      # (.config/nextest.toml profile ci). --all-targets keeps the
+      # terraphim-agents#91 coverage: binaries, examples and tests/*.rs.
+      - run: cargo install cargo-nextest --locked
+      - run: TERRAPHIM_SERVER_BIN=/tmp/terraphim_server_install/bin/terraphim_server cargo nextest run --workspace --all-targets --profile ci
+      # nextest does not run doctests.
+      - run: cargo test --workspace --doc --no-fail-fast
+      # #113: focused re-runs of the server-backed integration tests (also in
+      # the run above; kept for fast failure attribution).
+      - run: TERRAPHIM_SERVER_BIN=/tmp/terraphim_server_install/bin/terraphim_server cargo nextest run -p terraphim_agent --test cross_mode_consistency_test --test integration_tests --test kg_ranking_integration_test --profile ci
+      # #2171 / terraphim-clients#150: feature lanes invisible to default features.
+      - run: cargo nextest run -p terraphim_sessions --features enrichment --lib --profile ci
+      - run: cargo nextest run -p terraphim_sessions --all-features --profile ci
+      # #95 / #118: focused regression and repo guards (Rust tests, not shell
+      # steps: the runner allowlist rejects non-cargo programs).
+      - run: cargo nextest run -p terraphim_agent --test packaged_install_graph_regression --test ci_guards --profile ci
+  # Stage 3: coverage gate. Threshold only ever goes up (docs/ci-gap-analysis.md).
+  # --ignore-run-fail: the test job owns test failures; this job owns the
+  # percentage, so a red test must not hide the coverage number.
+  coverage:
+    needs: check
+    runs-on: terraphim-native
+    steps:
+      - run: rustup component add llvm-tools
+      - run: cargo install cargo-llvm-cov --locked
+      - run: cargo llvm-cov --workspace --lib --lcov --output-path lcov.info --ignore-run-fail
+      - run: cargo llvm-cov report --fail-under-lines 65
+  # Stage 4: UB gates on nightly (from the #252 UB_RUNBOOK). Miri covers the
+  # pure-computation crates only; tokio/reqwest paths are unsupported by Miri.
+  # terraphim_hooks' discovery tests spawn a process (posix_spawn), which Miri
+  # cannot emulate, and validation::tests::test_validate_latency runs 1000
+  # timed iterations that are meaningless (and very slow) under Miri, hence
+  # --skip discovery --skip latency.
+  ub-gates:
+    needs: check
+    runs-on: terraphim-native
+    steps:
+      - run: rustup toolchain install nightly --profile minimal -c miri -c rust-src
+      - run: cargo +nightly miri setup
+      - run: MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test -p terraphim_negative_contribution -p terraphim_command_runtime -p terraphim_hooks --lib -- --skip discovery --skip latency
+      - run: MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-tree-borrows" cargo +nightly miri test -p terraphim_negative_contribution -p terraphim_command_runtime -p terraphim_hooks --lib -- --skip discovery --skip latency

.github/workflows/ci.yml

@@ -1,4 +1,10 @@
 name: CI
+# Public mirror of .gitea/workflows/native-ci.yml (Refs #254). Same gate
+# order and the same cargo invocations; differences are limited to
+# provisioning (GitHub actions instead of cargo install on a self-hosted
+# runner), the multi-platform matrix, and the absence of the terraphim_server
+# backed integration lane, which needs the private registry. See
+# docs/ci-gap-analysis.md "Sync rule".
 on:
   push:
     branches: [main]
@@ -9,8 +15,12 @@ on:
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
+  # The 1.21.x terraphim family resolves from the private Gitea registry
+  # (see [patch.crates-io] in Cargo.toml). Same secret release-binaries uses.
+  CARGO_REGISTRIES_TERRAPHIM_TOKEN: ${{ secrets.CARGO_REGISTRIES_TERRAPHIM_TOKEN }}
 
 jobs:
+  # Stage 1: fast checks
   build:
     runs-on: ubuntu-latest
     # The workspace pins twelve private-registry deps (Cargo.toml lines
@@ -24,8 +34,9 @@ jobs:
       CARGO_REGISTRIES_TERRAPHIM_TOKEN: ${{ secrets.CARGO_REGISTRIES_TERRAPHIM_TOKEN }}
     steps:
       - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
+      - uses: dtolnay/rust-toolchain@master
         with:
+          toolchain: 1.97.1
           components: rustfmt, clippy
       - uses: Swatinem/rust-cache@v2
       - name: Install release signature verifier
@@ -41,8 +52,36 @@ jobs:
           python3 -m unittest tests.test_manifest_builder_compatibility -v
       - run: cargo fmt --all -- --check
       - run: cargo clippy --workspace --all-targets -- -D warnings
+      - run: cargo clippy -p terraphim_sessions --features enrichment -- -D warnings
+      - uses: taiki-e/install-action@cargo-deny
+      - run: cargo deny check
+
+  # Stage 2: tests on every release platform
+  test:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: 1.97.1
+      - uses: Swatinem/rust-cache@v2
+      - uses: taiki-e/install-action@nextest
       - run: cargo build --workspace
-      - run: cargo test --workspace --lib --no-fail-fast
+      # Lib and doc tests on all platforms. Integration tests that need a
+      # terraphim_server binary run only on native-ci.
+      - run: cargo nextest run --workspace --lib --profile ci
+      - run: cargo test --workspace --doc --no-fail-fast
+      # #2171: enrichment-feature test invocation.
+      - run: cargo nextest run -p terraphim_sessions --features enrichment --lib --profile ci
+      # #4325: zero-chunk smoke for terraphim_grep default features.
+      - run: cargo nextest run -p terraphim_grep --test default_feature_smoke --profile ci
+      # #95: isolated packaged install-graph regression.
+      - run: cargo nextest run -p terraphim_agent --test packaged_install_graph_regression --profile ci
       # GitHub-side home for the updater integration contracts (managed-mode
       # receipt/refusal, strict v2 manifest readers, R2 checksum fail-closed).
       # On Gitea these lanes live in .gitea/workflows/native-ci.yml, which is
@@ -138,3 +177,58 @@ jobs:
             bash "$t"
             echo "::endgroup::"
           done
+
+  # Stage 3: coverage gate (Linux only; threshold only ever increases).
+  # --ignore-run-fail: the test job owns test failures; this job owns the number.
+  coverage:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: 1.97.1
+          components: llvm-tools
+      - uses: Swatinem/rust-cache@v2
+      - uses: taiki-e/install-action@cargo-llvm-cov
+      - run: cargo llvm-cov --workspace --lib --lcov --output-path lcov.info --ignore-run-fail
+      - run: cargo llvm-cov report --fail-under-lines 65
+      - uses: actions/upload-artifact@v4
+        with:
+          name: lcov
+          path: lcov.info
+
+  # Stage 4: UB gates on nightly (from the #252 UB_RUNBOOK)
+  ub-gates:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@nightly
+        with:
+          components: miri, rust-src
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo +nightly miri setup
+      - run: MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test -p terraphim_negative_contribution -p terraphim_command_runtime -p terraphim_hooks --lib -- --skip discovery --skip latency
+      - run: MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-tree-borrows" cargo +nightly miri test -p terraphim_negative_contribution -p terraphim_command_runtime -p terraphim_hooks --lib -- --skip discovery --skip latency
+
+  # Stage 5: benchmark regression, informational on PRs (baselines from #253)
+  benchmarks:
+    if: github.event_name == 'pull_request'
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: 1.97.1
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo bench -p terraphim_grep --features code-search --bench hybrid_search -- --output-format bencher | tee bench-output.txt
+      - uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: cargo
+          output-file-path: bench-output.txt
+          alert-threshold: "120%"
+          comment-on-alert: true
+          fail-on-alert: false
+          auto-push: false

deny.toml

@@ -0,0 +1,101 @@
+# cargo-deny configuration (Refs #254).
+#
+# Policy: advisories and yanked crates block the pipeline; unmaintained and
+# notice-level advisories warn. Every `ignore` entry below carries the reason
+# and the issue that owns its removal. Ignores are temporary by construction:
+# remove the entry as soon as the named fix lands.
+
+[graph]
+all-features = true
+targets = [
+    "x86_64-unknown-linux-gnu",
+    "aarch64-unknown-linux-gnu",
+    "aarch64-apple-darwin",
+    "x86_64-apple-darwin",
+    "x86_64-pc-windows-msvc",
+]
+
+[advisories]
+version = 2
+db-urls = ["https://github.com/rustsec/advisory-db"]
+# TEMPORARY: warn until the Cargo.lock bump lands (spin 0.9.8 is yanked and is
+# fixed by `cargo update -p spin`). Flip back to "deny" in that commit.
+yanked = "warn"
+ignore = [
+    # --- Lockfile bumps pending (semver-compatible, `cargo update -p <crate>`).
+    # Held back until the UB audit #252 finishes its Phase 3 dynamic runs, so
+    # Miri/TSan results are attributed to one dependency set. Remove all six
+    # entries in the same commit as the Cargo.lock bump.
+    { id = "RUSTSEC-2026-0204", reason = "crossbeam-epoch 0.9.18 -> 0.9.21 via cargo update; blocked on #252 Phase 3" },
+    { id = "RUSTSEC-2026-0258", reason = "h2 0.4.15 -> 0.4.19 via cargo update; blocked on #252 Phase 3" },
+    { id = "RUSTSEC-2026-0190", reason = "anyhow 1.0.102 -> 1.0.104 via cargo update; blocked on #252 Phase 3" },
+    # The same Cargo.lock bump also clears the yanked spin 0.9.8 (cargo update -p spin).
+    # Unsound-class advisories cargo-audit reports for event-listener, lru, memmap2 and
+    # git2 (RUSTSEC-2026-0221/0253/0186/0183/0184) are informational in cargo-deny and
+    # are not encountered by this config; the first three are fixed by the same bump,
+    # git2 needs fff-search to move to a patched release (child issue of #254).
+    # --- quick-xml: two lockfile copies, both below the fixed 0.41.
+    { id = "RUSTSEC-2026-0195", reason = "quick-xml 0.37 (self_update via terraphim_update 1.20.2 registry) and 0.38 (opendal 0.54); fix is upstream in self_update and opendal. Child issue of #254" },
+    { id = "RUSTSEC-2026-0194", reason = "same crate and chain as RUSTSEC-2026-0195" },
+    # --- Requires an upstream or major-version change.
+    { id = "RUSTSEC-2026-0189", reason = "rmcp 0.9.1 -> >=1.4.0 is a major upgrade of terraphim_mcp_server's transport; DNS-rebinding affects the streamable HTTP server only. Tracked as a child of #254" },
+    # --- Unmaintained (warn-level; listed so the reason is recorded).
+    { id = "RUSTSEC-2025-0141", reason = "bincode 1.x via transitive deps; no maintained drop-in" },
+    { id = "RUSTSEC-2025-0119", reason = "number_prefix via indicatif" },
+    { id = "RUSTSEC-2024-0436", reason = "paste proc-macro; compile-time only" },
+    { id = "RUSTSEC-2024-0370", reason = "proc-macro-error; compile-time only" },
+]
+
+[licenses]
+version = 2
+confidence-threshold = 0.9
+allow = [
+    "MIT",
+    "MIT-0",
+    "Apache-2.0",
+    "Apache-2.0 WITH LLVM-exception",
+    "BSD-2-Clause",
+    "BSD-3-Clause",
+    "ISC",
+    "Zlib",
+    "0BSD",
+    "Unlicense",
+    "CC0-1.0",
+    "BSL-1.0",
+    "Unicode-3.0",
+    "CDLA-Permissive-2.0",
+    "MPL-2.0",
+    "bzip2-1.0.6",
+]
+# LGPL appears only as one option of a tri-licensed crate (r-efi: MIT OR
+# Apache-2.0 OR LGPL-2.1-or-later); cargo-deny picks an allowed option, so
+# no exception is needed.
+
+[[licenses.exceptions]]
+# html2md is GPL-3.0+ and reaches terraphim_agent and terraphim-cli
+# (both Apache-2.0) through terraphim_middleware (terraphim-ai registry).
+# This is an upstream licence-compatibility problem, not a local one: the
+# fix is for terraphim_middleware to replace html2md or gate it behind a
+# non-default feature. Recorded here so the gate stays green while the
+# upstream issue is open; remove when terraphim_middleware drops html2md.
+allow = ["GPL-3.0-or-later", "GPL-3.0"]
+name = "html2md"
+
+[bans]
+multiple-versions = "warn"
+wildcards = "deny"
+highlight = "all"
+deny = [
+    { name = "openssl-sys", reason = "workspace policy is rustls everywhere; native TLS breaks the static release-binaries matrix" },
+]
+
+[sources]
+unknown-registry = "deny"
+unknown-git = "deny"
+allow-registry = [
+    "https://github.com/rust-lang/crates.io-index",
+    "sparse+https://git.terraphim.cloud/api/packages/terraphim/cargo/",
+]
+allow-git = [
+    "https://github.com/rustls/webpki",
+]