Measure test coverage in CI and cover the app and tools

.github/workflows/ci.yml

@@ -1,9 +1,10 @@
 # Copyright (c) 2026 Jim Schreckengast
 # SPDX-License-Identifier: MIT OR Apache-2.0
 #
-# Build and test on every platform, build the browser version, check formatting and lint
-# with clippy, build the crate from the files crates.io would get, check licenses,
-# advisories and attributions, and fuzz the loading of weights files.
+# Build and test on every platform, measure how much of the crate the tests run, build
+# the browser version, check formatting and lint with clippy, build the crate from the
+# files crates.io would get, check licenses, advisories and attributions, and fuzz the
+# loading of weights files.
 name: CI
 
 on:
@@ -34,6 +35,27 @@ jobs:
       - run: cargo build --locked --release --all-targets
       - run: cargo test --locked --release
 
+  # The share of the crate's lines the tests run, every file counted: the app's panels
+  # and the developer tools in src/bin too. Fails below 80%.
+  coverage:
+    name: Coverage
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
+        with:
+          components: llvm-tools-preview
+      - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+      - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
+        with:
+          tool: cargo-llvm-cov@0.9.1
+      # One test at a time: code built for coverage counts what it runs in counters every
+      # thread updates, so threads running the same code at once slow one another down
+      # many times over (tests/common/mod.rs).
+      - run: cargo llvm-cov --locked --release --fail-under-lines 80 -- --test-threads=1
+
   web:
     name: Web (wasm32)
     runs-on: ubuntu-22.04

CONTRIBUTING.md

@@ -59,6 +59,23 @@ cargo clippy --locked --release --target wasm32-unknown-unknown --bin intersecti
 cargo deny --locked check
 ```
 
+CI also measures how much of the crate the tests run, with
+[cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov), and fails if they run less
+than 80% of its lines. Every file counts, including the app's panels and the developer
+tools in `src/bin`, so new code needs tests that reach it. To measure locally:
+
+```bash
+rustup component add llvm-tools-preview
+cargo install cargo-llvm-cov --locked
+cargo llvm-cov --locked --release --fail-under-lines 80 -- --test-threads=1
+```
+
+The tests run one at a time, as in CI: code built for coverage slows down many times
+over when several threads run it at once. For the same reason, a test that runs many
+simulations at once should start them with `spawn` from
+[tests/common/mod.rs](tests/common/mod.rs), which runs them in turn when measuring
+coverage. Add `--open` to see, file by file, the lines no test runs.
+
 If you add or update a dependency, also regenerate the third-party license list with
 `packaging/licenses.sh` (it needs `cargo-about` 0.9.1). CI fails if
 [THIRD_PARTY_LICENSES.md](THIRD_PARTY_LICENSES.md) is out of date.
@@ -95,9 +112,11 @@ pass that file's path in place of the two folders.
   check it, and a bug fix comes with a test that fails without the fix. Put tests that
   use the public API in [tests/](tests/) (see
   [tests/pedestrian_rule.rs](tests/pedestrian_rule.rs) for an example) and narrower ones
-  in a `#[cfg(test)]` module next to the code. Use fixed seeds so tests are
-  deterministic, and keep them fast enough to run on every pull request. Changes to
-  documentation, UI text or styling don't need tests.
+  in a `#[cfg(test)]` module next to the code. Test the app the way
+  [tests/gui.rs](tests/gui.rs) does: without a window, through the names its controls
+  give screen readers. Use fixed seeds so tests are deterministic, and keep them fast
+  enough to run on every pull request. Changes to documentation, UI text or styling
+  don't need tests.
 * **Report results with your change.** If it changes how traffic behaves, include before
   and after numbers from `cargo run --release --bin bench all 20 6` (and the corridor or
   sensor study where relevant), and update the tables in the README and

Cargo.toml

@@ -40,6 +40,16 @@ serde_json = "1.0.154"
 wasm-bindgen-futures = "0.4.79"
 web-time = "1.1.0"
 
+# The GUI tests (tests/gui.rs) run the app without a window and drive it through its
+# accessibility tree.
+[dev-dependencies]
+egui_kittest = { version = "0.37", features = ["eframe"] }
+
+# cargo-llvm-cov sets cfg(coverage) when it measures how much of the crate the tests run
+# (see tests/common).
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ["cfg(coverage)"] }
+
 [profile.release]
 opt-level = 3
 debug = false

README.md

@@ -60,7 +60,7 @@ cargo run --release --example lpi                 # pedestrian safety treatments
 cargo run --release --example fya                 # left-turn phasing modes
 cargo run --release --example satflow             # calibration against HCM saturation flow
 cargo run --release --example cooperative         # capacity of the signal-free cooperative crossing
-cargo test --release                              # safety + throughput regression tests
+cargo test --release                              # regression tests: safety, throughput, app, tools
 ```
 
 The browser version is the same app compiled to WebAssembly. Build and serve it locally
@@ -473,7 +473,7 @@ src/rl/         MLP + Adam, Double DQN with replay
 src/gui/        eframe/egui app: renderer, charts, panels, self-screenshot (TS_SHOT=…)
 src/bin/        bench (comparisons, sensor study), train (RL training)
 weights/        trained DQN and hybrid weights per layout
-tests/          safety + throughput regression tests
+tests/          regression tests: safety, throughput, the app (headless) and the tools
 examples/       corridor, lpi, fya, satflow, preempt, sweep (multi-seed safety sweep), cooperative
 docs/           architecture.md (how the code fits together, with a tour of the app),
                 insights.md (every finding), cooperative-intersections.md (design notes for the

THIRD_PARTY_LICENSES.md

@@ -3841,7 +3841,7 @@ DEALINGS IN THE SOFTWARE.
 
 Used by:
 
-* toml_datetime 1.1.2+spec-1.1.0 (https://github.com/toml-rs/toml)
+* toml_datetime 1.1.3+spec-1.1.0 (https://github.com/toml-rs/toml)
 * toml_edit 0.25.17+spec-1.1.0 (https://github.com/toml-rs/toml)
 * toml_parser 1.1.5+spec-1.1.0 (https://github.com/toml-rs/toml)
 

docs/architecture.md

@@ -108,7 +108,7 @@ flowchart TB
     SIM <-->|"Ctx in, stage out"| CTRL
     CTRL --> RL
     CTRL --> W
-    STUDY -->|"fuzz/: rl::dqn"| RL
+    STUDY -->|"fuzz/, tests/: rl"| RL
     RL -.->|"random numbers"| SIM
     classDef ui fill:#fff3e0,stroke:#c05621,color:#3b1d06
     classDef sim fill:#e7f5ec,stroke:#2f855a,color:#123524
@@ -132,7 +132,7 @@ Each front end uses only the library's public modules:
 | `bin/bench.rs` | `lib.rs` (`scenario`, `run_scenario`), `sim` and `control` |
 | `bin/train.rs` | `sim` and `control` (it drives `Sim` itself) |
 | `examples/` | `sim` and `control`; `make_icons` uses only `gui::shot` |
-| `tests/` | `sim` and `control` |
+| `tests/` | `sim`, `control`, `rl` and `gui` (the app, run without a window); `bench` and `train` as programs |
 | `fuzz/` | `sim`, `control` and `rl::dqn` |
 
 ### 2.1 Responsibilities by layer
@@ -214,7 +214,8 @@ src/
   bin/train.rs      headless training of the Deep RL and Hybrid networks
 examples/           studies: cooperative, corridor, fya, lpi, preempt, satflow, sweep,
                     plus make_icons
-tests/              safety, cooperative, pedestrian_rule, weights_files, green_wave
+tests/              safety, cooperative, pedestrian_rule, weights_files, green_wave,
+                    networks, gui, scripted, tools, and common/ (shared by the tests)
 fuzz/               cargo-fuzz target for weights files (a workspace of its own)
 weights/            trained networks (JSON), also compiled into the binary
 web/index.html      page that hosts the browser build (Trunk)
@@ -235,7 +236,7 @@ docs/               this document, insights.md, cooperative-intersections.md (wi
 | `src/bin/` | 2 | 590 |
 | `src/lib.rs`, `src/main.rs` | 2 | 78 |
 | `examples/` | 8 | 575 |
-| `tests/` | 5 | 873 |
+| `tests/` | 10 | 2,557 |
 | `fuzz/fuzz_targets/` | 1 | 54 |
 
 | Target | Built from | Purpose |
@@ -250,7 +251,8 @@ docs/               this document, insights.md, cooperative-intersections.md (wi
 
 The crate is edition 2024 and has four dependencies: `eframe` and `egui` 0.37, `serde`
 and `serde_json`. The browser build adds two more, `web-time` and
-`wasm-bindgen-futures`. The crate has no features and no dev-dependencies. The package
+`wasm-bindgen-futures`. The crate has no features. Its one dev-dependency,
+`egui_kittest` 0.37, runs the app without a window for the tests (§10.3). The package
 published to crates.io holds only the library, the app, the weights files, the icon, the
 licenses and the README, so `bench`, `train`, the examples and the tests are not in it.
 
@@ -3085,9 +3087,9 @@ The examples are the studies behind `docs/insights.md` and
 
 ### 10.3 Tests
 
-`cargo test` runs five integration test files and the unit tests in `sim/coop.rs`,
+`cargo test` runs nine integration test files and the unit tests in `sim/coop.rs`,
 `sim/signal.rs` and `gui/mod.rs`. CI runs them in release mode on Linux, macOS and
-Windows (§11).
+Windows, and measures how much of the crate they cover (§11).
 
 | File | Tests | What they require |
 |---|---:|---|
@@ -3096,6 +3098,10 @@ Windows (§11).
 | `tests/pedestrian_rule.rs` | 4 | Deep RL's shield serves a crosswalk whose pedestrians have waited longer than `PED_MAX_WAIT`, longest wait first, and leaves shorter waits to the controller; the reward counts waiting pedestrians |
 | `tests/weights_files.rs` | 6 | The shipped weights load, misshapen networks are rejected, a Hybrid without a fitting network keeps its base tuning and says so, Hybrid files saved with a score that isn't finite load again, a copy that doesn't load is skipped for the next one and networks that overflow don't crash (§8.7) |
 | `tests/green_wave.rs` | 2 | The green wave runs every signal of a corridor fixed-time, and ending it gives back the controllers they ran before, except where another was chosen during the wave, without collisions; a single intersection isn't coordinated |
+| `tests/networks.rs` | 7 | Backpropagation agrees with the gradient measured numerically, Adam fits a line, the shape check rejects every malformed network (sizes whose product overflows included), the target network moves part way toward the online one, the action mask is respected, Double DQN learns which action pays in which state and the replay memory keeps the latest 60,000 transitions |
+| `tests/gui.rs` | 16 | The app, run without a window by `egui_kittest` and driven through its accessibility tree, the keyboard and the pointer: every panel opens and every control has a name; Space pauses and Step advances one second; every layout, control mode and controller runs, side by side too and along a corridor with the green wave; the demand, sensor and operations settings apply; the learning controllers show what they learn; every overlay and coloring draws; vehicles and pedestrians placed by hand, movements an approach lacks refused; a clicked vehicle's details; pan, zoom and *Fit view*; both benchmark windows; *About* |
+| `tests/scripted.rs` | 8 | Each `TS_*` variable of the scripted mode sets up its scene, a scripted screenshot stops the clock at its moment and screenshots are valid PNG files. The app reads the variables as it starts, so each scene opens in a process of its own |
+| `tests/tools.rs` | 7 | `bench` compares every strategy on every layout without collisions, takes each arrival pattern and runs the sensor study; `train` trains, saves and resumes both learning controllers, in a scratch directory with its own home directory |
 | `src/sim/coop.rs` | 4 | Approach profiles arrive on time and at speed, a short run-up enters slower but on time, the gap policies give the gaps in §6.11 and the following law holds its time gap |
 | `src/sim/signal.rs` | 1 | The signal history records a change's yellow and all-red between the two greens, with no gaps |
 | `src/gui/mod.rs` | 1 | The safety monitors' margin over the gap keeps its sign |
@@ -3117,6 +3123,7 @@ flowchart TB
     PR["<b>Pull request</b>"]
     CI["<b>Checks</b><br/>CI, CodeQL and DCO;<br/>10 are required"]
     FZ["Fuzz (weights), 60 s:<br/>reported, not required"]
+    CV["Coverage, at least 80%<br/>of lines: reported,<br/>not required"]
     M["<b>Merge to main</b><br/>branch up to date"]
     PG["<b>Pages</b>: the browser<br/>build, deployed"]
     SC["<b>Scorecard</b><br/>also weekly"]
@@ -3126,13 +3133,14 @@ flowchart TB
     C["<b>publish-crate</b><br/>cargo publish to<br/>crates.io"]
     PR --> CI --> M --> T --> R --> P --> C
     PR -.-> FZ
+    PR -.-> CV
     M --> PG
     M --> SC
     classDef ui fill:#fff3e0,stroke:#c05621,color:#3b1d06
     classDef data fill:#f1f3f5,stroke:#5f6b7a,color:#1f2933
     classDef safe fill:#fdecec,stroke:#c53030,color:#4a0f0f
     class PR,M,T,P ui
-    class CI,FZ safe
+    class CI,FZ,CV safe
     class PG,SC,R,C data
 ```
 
@@ -3151,6 +3159,13 @@ date:
 
 * **Fuzz (weights)** in `ci.yml` runs the fuzz target for 60 s from its seeds on the
   nightly toolchain, and uploads any crash. It isn't required.
+* **Coverage** in `ci.yml` runs the tests on Linux with
+  [cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov) and fails if they run less
+  than 80% of the crate's lines, every file counted (CONTRIBUTING.md says how to measure
+  it locally). The tests run one at a time, and under `cfg(coverage)` the ones that run
+  many simulations at once run them in turn: code built for coverage counts what it runs
+  in counters that every thread updates, so threads running the same code at once slow
+  one another down many times over. It isn't required.
 * **`pages.yml`** builds the browser version with Trunk on every push to `main` and
   deploys it to GitHub Pages.
 * **`release.yml`** runs on a `v*` tag, or by hand to build unsigned packages:

tests/common/mod.rs

@@ -0,0 +1,19 @@
+// Copyright (c) 2026 Jim Schreckengast
+// SPDX-License-Identifier: MIT OR Apache-2.0
+
+//! Shared by the integration tests that run many simulations at once.
+
+use std::thread::JoinHandle;
+
+/// Run `job` on a thread of its own, except when measuring test coverage, which runs it
+/// right away. Code built for coverage counts what it runs in counters that all threads
+/// update, so threads running the same code at once slow one another down many times
+/// over (cargo-llvm-cov sets `cfg(coverage)`; see CONTRIBUTING.md).
+pub fn spawn<T: Send + 'static>(job: impl FnOnce() -> T + Send + 'static) -> JoinHandle<T> {
+    if cfg!(coverage) {
+        let done = job();
+        std::thread::spawn(move || done)
+    } else {
+        std::thread::spawn(job)
+    }
+}

tests/cooperative.rs

@@ -14,6 +14,8 @@ use intersection_lab::sim::stats::Summary;
 use intersection_lab::sim::vtypes::VKind;
 use intersection_lab::sim::{ControlMode, Sim};
 
+mod common;
+
 /// A crossing with `n_left` left-turn lanes per approach and `total` veh/h spread evenly
 /// over the four approaches, with the safety checks on.
 fn crossing(n_left: usize, total: f64, cars_only: bool, policy: GapPolicy, seed: u64) -> Sim {
@@ -150,7 +152,7 @@ fn every_gap_policy_is_safe_under_heavy_demand() {
         for (n_left, total) in [(2, 18_000.0), (1, 12_000.0)] {
             for cars in [false, true] {
                 for seed in 31..34 {
-                    jobs.push(std::thread::spawn(move || {
+                    jobs.push(common::spawn(move || {
                         let mut sim = crossing(n_left, total, cars, policy, seed);
                         sim.run_for(1200.0);
                         let fleet = if cars { "cars" } else { "mixed" };
@@ -205,7 +207,7 @@ fn bursts_heavy_vehicles_and_uneven_demand_are_served_safely() {
     let jobs: Vec<_> = cases
         .into_iter()
         .map(|(name, d, limits)| {
-            std::thread::spawn(move || {
+            common::spawn(move || {
                 let mut sim =
                     Sim::with_demand(build_cooperative(2), ControlMode::Cooperative, CtrlKind::Fixed, d, 41);
                 sim.check_safety = true;
@@ -259,7 +261,7 @@ fn weaving_one_at_a_time_carries_far_less() {
     // quicker than waiting for a batch; but every switch between crossing streams costs
     // a full safety gap, which a batch pays once, so it saturates far below the batches.
     let run = |sched: Scheduling, total: f64, seed: u64| {
-        std::thread::spawn(move || {
+        common::spawn(move || {
             let mut sim = crossing(2, total, false, GapPolicy::FailSafe, seed);
             sim.set_scheduling(sched);
             sim.run_for(120.0 + 900.0);
@@ -305,7 +307,7 @@ fn dual_left_design_serves_18000_vehicles_per_hour() {
     // arrives is served, nobody waits long, and the monitors see nothing wrong.
     let jobs: Vec<_> = (0..3)
         .map(|seed| {
-            std::thread::spawn(move || {
+            common::spawn(move || {
                 let mut sim = crossing(2, 18_000.0, false, GapPolicy::FailSafe, 7000 + seed);
                 sim.run_for(120.0 + 1200.0);
                 let s = summary(&sim, 120.0);
@@ -348,7 +350,7 @@ fn oversaturated_capacity_is_about_19000_vehicles_per_hour() {
 #[test]
 fn single_left_lanes_fall_short_of_the_design_demand() {
     let run = |n_left: usize| {
-        std::thread::spawn(move || {
+        common::spawn(move || {
             let mut sim = crossing(n_left, 16_000.0, false, GapPolicy::FailSafe, 3);
             sim.run_for(900.0);
             (summary(&sim, 120.0).throughput_vph, queued(&sim))