diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 230544bfa..aa3a6ed7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -539,8 +539,10 @@ jobs: # in the merge queue the selection is forced to `--workspace`, so post-merge # `main` and every queued PR run the full suite — a required check is never # starved and the queue cannot stall. The full Windows suite - # (the windows-debug job) also runs the whole workspace on every change, so it - # backstops any crate a graph-based selection could miss (e.g. an integration + # (the windows-debug job) also runs the whole workspace on every Rust change + # (its `heavy` filter matches every `.rs` file), and so does the Linux + # `coverage` job for every crate but e2e-cucumber, so they + # backstop any crate a graph-based selection could miss (e.g. an integration # test that drives another crate only at runtime). # # This job is deliberately left off sccache, unlike the other compile-heavy @@ -623,12 +625,14 @@ jobs: - name: Test rocm-core with e2e-test-hooks # The simulated-host-root tests (`hardware_root.rs`, and the planted-root - # probe tests in `examine.rs`) only exist with this feature, which no - # other unit-test, nextest or clippy invocation enables. Without this - # step they are not even compiled in CI, so a probe reverted to a raw - # path would pass everything automated. Runs only when rocm-core is in - # the affected selection (or the whole workspace is), since nothing - # outside the crate can change these results. + # probe tests in `examine.rs`) only exist with this feature, which the + # plain nextest step above does not enable. The `rocm` step below + # turns it on in rocm-core too but runs only rocm's own tests, and the + # feature-on clippy step compiles these tests without running them, so + # without this step a probe reverted to a raw path would pass + # everything automated. Runs only when rocm-core is in the affected + # selection (or the whole workspace is), since nothing outside the + # crate can change these results. # # This is a sample, not full coverage of the routed reads. The probe # tests assert only the KFD topology (GPU node count and gfx target), @@ -658,6 +662,26 @@ jobs: || contains(format('{0} ', steps.sel.outputs.sel), '-p rocm-core ')) run: cargo nextest run --locked -p rocm-core --lib --features e2e-test-hooks + - name: Test rocm with e2e-test-hooks + # Runs the whole rocm suite with the feature on, not only the tests + # that exist only with it (the low-VRAM hook in `gpu_policy.rs`, and + # the `system_packages.rs` tests that plant a simulated host root), + # which the feature-on clippy step compiles but only this step runs. + # Tests gated on the feature being off (`cfg(not(..))`, as in + # `dash.rs`) are not built here; the plain nextest step runs them. + # The selection gate is the rocm-core step's: the trailing space stops + # `-p rocm-core` from satisfying a check for `-p rocm `. The step uses + # nextest so that each test is its own process: the feature-only tests + # set process-wide variables the hooks read (`ROCM_E2E_FORCE_LOW_VRAM`, + # and the simulated host root) while holding `ScopedTestEnv`'s lock, + # but the hooks read them without it, so under a threaded `cargo test` + # another test reaching a hook in that window could see them. + if: >- + needs.changes.outputs.rust == 'true' + && (steps.sel.outputs.sel == '--workspace' + || contains(format('{0} ', steps.sel.outputs.sel), '-p rocm ')) + run: cargo nextest run --locked -p rocm --features rocm/e2e-test-hooks + # The Windows lane is two parallel jobs (`windows-debug`, `windows-release`) # joined by `windows-build-and-test` below. One serial job was the longest pole # of the whole run (it alone decided when CI went green); the debug side @@ -777,9 +801,12 @@ jobs: # of every platform conditional and never sees the `cfg(windows)` side. # Windows-gated code was therefore outside the warnings-as-errors gate # entirely — not merely under-linted — so a lint could land there and no - # check in the set could fail. This mirrors the two invocations the Linux - # `clippy` job runs; the second is separate because the `e2e` target sets - # `test = false`, so `--all-targets` does not build it. + # check in the set could fail. This mirrors the Linux `clippy` job's + # plain workspace pass and its e2e-harness pass. The e2e-harness pass is + # separate because the `e2e` target sets `test = false`, so + # `--all-targets` does not build it. The `e2e-test-hooks` feature-on + # pass runs on Linux only, so the feature's code as compiled for Windows + # is not linted by any CI clippy step. # # Runs before the build so a lint failure costs seconds rather than the # full build and test cycle. @@ -1447,6 +1474,17 @@ jobs: if: needs.changes.outputs.rust == 'true' run: cargo clippy --locked --workspace --all-targets -- -D warnings + # A second pass, not a replacement: each `e2e-test-hooks` seam has a + # feature-off half (the `cfg(not(..))` fallbacks in, e.g., comfyui.rs, + # service_control.rs, amd_smi.rs and hardware_root.rs) that only the run + # above compiles, and a feature-on half, including the tests that exist + # only with the feature, that only this one does. Enabling it on `rocm` + # turns it on in rocm-core and the engine crates as well, so the + # rocm-core and Lemonade seams are linted here too. + - name: Clippy (e2e-test-hooks) + if: needs.changes.outputs.rust == 'true' + run: cargo clippy --locked --workspace --all-targets --features rocm/e2e-test-hooks -- -D warnings + # The e2e-cucumber `e2e` target sets `test = false` (so nextest/`cargo test # --all-targets` skip the custom harness), which also excludes it from # `clippy --all-targets` above — leaving the ~1.6k lines of harness + step diff --git a/AGENTS.md b/AGENTS.md index 6b972da2d..7d3671ef4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -399,9 +399,12 @@ Minimum quality gate before upstream-ready status: ```bash cargo test --workspace --all-targets cargo clippy --workspace --all-targets -- -D warnings +cargo clippy --workspace --all-targets --features rocm/e2e-test-hooks -- -D warnings cargo xtask smoke ``` +The feature-on test steps are not part of this minimum gate: CI's `test` job runs them, and the pre-push hooks build with `e2e-test-hooks` off, so when a change touches code or tests behind `e2e-test-hooks`, also run the commands in [CI test selection](docs/testing.md#ci-test-selection) by hand. + Deleting or weakening a test is a change in its own right: every crate has a committed coverage floor in `coverage-floors.toml`, and `cargo xtask coverage` fails when one drops below it. If a drop is legitimate, re-bless with `cargo xtask coverage --bless` and say in diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b9adc15e..1c4b0147c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,9 +46,9 @@ prek install # fast checks on every commit prek install -t pre-push # heavier checks on push (clippy + tests of the crates the push affects) ``` -`prek` runs the same checks locally that CI enforces: `cargo fmt`, `clippy`, `cargo test`, `ruff` (Python), `shellcheck` (shell), PowerShell syntax, license headers (`hawkeye`), markdown links (`lychee`), the module map (`cargo xtask check-architecture-doc`), and the generated manifests (`MANIFEST.md`, `THIRD_PARTY_NOTICES.txt`). +`prek` runs the same checks locally that CI enforces: `cargo fmt`, `clippy`, `cargo test`, `ruff` (Python), `shellcheck` (shell), PowerShell syntax, license headers (`hawkeye`), markdown links (`lychee`), the module map (`cargo xtask check-architecture-doc`), and the generated manifests (`MANIFEST.md`, `THIRD_PARTY_NOTICES.txt`). Its `clippy` and `cargo test` build with the `e2e-test-hooks` feature off, so CI's feature-on clippy step and the feature-on test steps run only in CI, or by hand with the commands in `docs/testing.md`. -The pre-push hooks (`cargo clippy`, the e2e harness clippy and `cargo test`) check only the crates your push affects: the crates whose files changed plus every crate that depends on them, selected the way CI's `cargo xtask affected` does. They check the whole workspace when a change cannot be confined to specific crates (the lockfile, the root manifest, the toolchain file, workflow files, a doc an xtask test reads, or a path no crate owns), and when the affected crates cannot be determined at all, for example because the remote branch tip is not fetched locally or the pushed range lists no changed files. They also add the crates whose tests read files from outside their own crate: any Rust file adds `xtask`, whose contract tests scan every source file, and `README.md`, `docs/` and `skills/` add `rocm` and `xtask`. Each step prints the scope it chose and why. A step that already passed on this exact content is skipped, and says so: after a green run it records a stamp keyed on the git tree hash of `HEAD`, the exact cargo command (so a pass for a narrow selection never covers a wider one), the toolchain and the `CARGO_*` and `RUST*` environment, under the clone's `.git` directory where every worktree shares it, so a retried push, a manual run before the push or an amend that only rewords the message does not pay for the same checks twice. Stamps are only used and written when the working tree is clean (cargo checks the working tree, not the pushed commit, and an untracked file can hide a missing `git add`), expire after seven days, and do not cover your user-level cargo configuration, installed tools, files git ignores or flaky tests. A skip shows that this tree passed those checks earlier, not that they ran during this push; the `cargo test --workspace` and `cargo clippy --workspace` commands you run by hand are unaffected, and `--rerun` gives a fresh run when you want proof the checks ran for this push. `cargo xtask pre-push-check --rerun`, or `ROCM_PRE_PUSH_RERUN=1` for the hook, ignores them, drops this step's earlier pass so a failed rerun cannot leave it to skip the next run, and records a fresh pass when the step succeeds; `SKIP=` still skips a hook outright, and the commit-signature hook runs on every push. A push that changes no Rust file runs none of the cargo steps, whatever else it changes (a lockfile-only push included). A green hook is an early warning and not the gate: on pull requests CI lints the whole workspace, but its Linux test job runs only the affected crates, and the whole workspace's tests run in the Windows debug job, in the merge queue and on `main`. To see what a push would check without compiling anything, run `cargo xtask pre-push-check test --dry-run --base origin/main` (the steps are `clippy`, `clippy-e2e` and `test`). +The pre-push hooks (`cargo clippy`, the e2e harness clippy and `cargo test`) check only the crates your push affects: the crates whose files changed plus every crate that depends on them, selected the way CI's `cargo xtask affected` does. They check the whole workspace when a change cannot be confined to specific crates (the lockfile, the root manifest, the toolchain file, workflow files, a doc an xtask test reads, or a path no crate owns), and when the affected crates cannot be determined at all, for example because the remote branch tip is not fetched locally or the pushed range lists no changed files. They also add the crates whose tests read files from outside their own crate: any Rust file adds `xtask`, whose contract tests scan every source file, and `README.md`, `docs/` and `skills/` add `rocm` and `xtask`. Each step prints the scope it chose and why. A step that already passed on this exact content is skipped, and says so: after a green run it records a stamp keyed on the git tree hash of `HEAD`, the exact cargo command (so a pass for a narrow selection never covers a wider one), the toolchain and the `CARGO_*` and `RUST*` environment, under the clone's `.git` directory where every worktree shares it, so a retried push, a manual run before the push or an amend that only rewords the message does not pay for the same checks twice. Stamps are only used and written when the working tree is clean (cargo checks the working tree, not the pushed commit, and an untracked file can hide a missing `git add`), expire after seven days, and do not cover your user-level cargo configuration, installed tools, files git ignores or flaky tests. A skip shows that this tree passed those checks earlier, not that they ran during this push; the `cargo test --workspace` and `cargo clippy --workspace` commands you run by hand are unaffected, and `--rerun` gives a fresh run when you want proof the checks ran for this push. `cargo xtask pre-push-check --rerun`, or `ROCM_PRE_PUSH_RERUN=1` for the hook, ignores them, drops this step's earlier pass so a failed rerun cannot leave it to skip the next run, and records a fresh pass when the step succeeds; `SKIP=` still skips a hook outright, and the commit-signature hook runs on every push. A push that changes no Rust file runs none of the cargo steps, whatever else it changes (a lockfile-only push included). A green hook is an early warning and not the gate: on pull requests CI lints the whole workspace, but its Linux test job runs only the affected crates, and the whole workspace's tests run in the Windows debug job on every Rust change, in the merge queue and on `main`, and the Linux coverage job runs every crate's tests but `e2e-cucumber`'s on every Rust change. To see what a push would check without compiling anything, run `cargo xtask pre-push-check test --dry-run --base origin/main` (the steps are `clippy`, `clippy-e2e` and `test`). The manifest hooks only run when you change the dependency graph. They *rewrite* the generated file instead of only reporting that it is stale. When that happens, the commit stops so you can re-stage the refreshed file. diff --git a/Cargo.lock b/Cargo.lock index ed1f160cf..b11d61064 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6132,6 +6132,7 @@ dependencies = [ "e2e-report", "flate2", "proc-macro2", + "quote", "regex", "rocm-core", "semver", diff --git a/docs/architecture.md b/docs/architecture.md index b97561e7a..8be35d63c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -747,7 +747,7 @@ Repository tasks, run through the workspace alias `cargo xtask `: relea | [`eval_fixtures.rs`](../xtask/src/eval_fixtures.rs) | `cargo xtask eval-fixtures`: regenerates or checks the recorded `rocm` output the rocm-doctor skill evals replay, rendered from the compiled catalog. | flat | | [`lychee.rs`](../xtask/src/lychee.rs) | `cargo xtask lychee`: runs lychee with [`lychee.toml`](../lychee.toml) over the working tree, the local counterpart of the `docs-links` CI job. | flat | | [`manifest.rs`](../xtask/src/manifest.rs) | `cargo xtask manifest`: regenerates or checks the Cargo dependency table in [`MANIFEST.md`](../MANIFEST.md) from `cargo metadata`. | flat | -| [`module_size.rs`](../xtask/src/module_size.rs) | `cargo xtask check-module-size`: fails when a source file exceeds `MAX_PRODUCTION_LINES` production lines, or when a `.rs` file under a crate's `src/` is declared by no `mod`. | flat | +| [`module_size.rs`](../xtask/src/module_size.rs) | `cargo xtask check-module-size`: fails when a source file exceeds `MAX_PRODUCTION_LINES` production lines, or when a `.rs` file under a crate's `src/` is declared by no `mod`. Also provides a test-only walk of every package's module tree that [`workflow_contract.rs`](../xtask/src/workflow_contract.rs) reuses. | flat | | [`package.rs`](../xtask/src/package.rs) | `cargo xtask package`: builds the release bundle directory from built binaries, its archive, the SHA-256 sidecar and, when a key is configured, a detached signature. | flat | | [`paths.rs`](../xtask/src/paths.rs) | Workspace paths shared by the tasks: the workspace root, the active `target/` directory, release binary locations and platform executable names. | flat | | [`powershell.rs`](../xtask/src/powershell.rs) | `cargo xtask powershell-lint`: runs PSScriptAnalyzer over `.ps1` files through `pwsh` or Windows PowerShell, which also reports syntax errors. | flat | @@ -764,11 +764,11 @@ Repository tasks, run through the workspace alias `cargo xtask `: relea | [`tpn.rs`](../xtask/src/tpn.rs) | `cargo xtask tpn`: regenerates or checks [`THIRD_PARTY_NOTICES.txt`](../THIRD_PARTY_NOTICES.txt) through cargo-about with the committed config and template. | flat | | [`verify_commits.rs`](../xtask/src/verify_commits.rs) | `cargo xtask verify-commits`: checks that every commit in a range is signed and carries a DCO `Signed-off-by` trailer, or only that signing is configured. | flat | | [`verify_pinned_keys.rs`](../xtask/src/verify_pinned_keys.rs) | `cargo xtask verify-pinned-keys`: checks that the public keys pinned in the installers and the CLI match the canonical keys under `docs/keys/`. | flat | -| [`workflow_contract.rs`](../xtask/src/workflow_contract.rs) | Source-scan tests over the GitHub workflow files and CI scripts: lane placement and concurrency groups, runner labels, timeouts, permissions and other CI invariants. | flat | +| [`workflow_contract.rs`](../xtask/src/workflow_contract.rs) | Source-scan tests over the GitHub workflow files and CI scripts: lane placement and concurrency groups, runner labels, timeouts, permissions and other CI invariants, including that CI lints and runs every crate's `e2e-test-hooks`-only tests, found by walking the module tree [`module_size.rs`](../xtask/src/module_size.rs) walks. | flat | Boundaries: -- The eight `*_contract.rs` files define no command: each is one `#[cfg(test)]` module run by `cargo test -p xtask`, and [`rust_source.rs`](../xtask/src/rust_source.rs) is declared test-only in [`main.rs`](../xtask/src/main.rs). The pull-request test selection narrows to the crates a change touches, so a change confined to another crate may not select them there; pushes to `main` and the merge queue run the whole workspace. +- The eight `*_contract.rs` files define no command: each is one `#[cfg(test)]` module run by `cargo test -p xtask`, and [`rust_source.rs`](../xtask/src/rust_source.rs) is declared test-only in [`main.rs`](../xtask/src/main.rs). The pull-request test selection narrows to the crates a change touches, so a change confined to another crate may not select them there; the `windows-debug` job still runs the whole workspace's tests on every Rust change, as do pushes to `main` and the merge queue, and the Linux `coverage` job runs every crate's tests but `e2e-cucumber`'s. - [`affected.rs`](../xtask/src/affected.rs) is the one crate selection used by both CI's affected-crates job and [`prepush.rs`](../xtask/src/prepush.rs), which only widens it with `CROSS_CRATE_READS`; a change to how files are attributed to crates changes both. - [`paths.rs`](../xtask/src/paths.rs) defines the workspace root, the active target directory and binary file names, honouring `CARGO_TARGET_DIR` and a cross-compilation target; a new task uses it rather than recomputing them. diff --git a/docs/testing.md b/docs/testing.md index a21272ebc..3f4c5634c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,12 +21,19 @@ also what the `windows-debug` job of the required `windows-build-and-test` lane does. See [Tests that touch the environment](#tests-that-touch-the-environment) before writing a test that sets an environment variable. -Run clippy with warnings as errors: +Run clippy with warnings as errors, once with the `e2e-test-hooks` feature off +and once with it on: ```bash cargo clippy --workspace --all-targets -- -D warnings +cargo clippy --workspace --all-targets --features rocm/e2e-test-hooks -- -D warnings ``` +Each run compiles code the other never sees: a test hook's `cfg(not(..))` +fallback only with the feature off, and the hook itself, with the tests that +exist only alongside it, only with the feature on. The CI `clippy` job runs both. +The Windows clippy steps in the `windows-debug` job lint with the feature off only. + ### Source file size No source file may exceed 1,500 production lines. Production lines leave out test-gated @@ -258,13 +265,15 @@ stops being needed, or whose file moves. A new field reuses the helper; a new exception is a reviewed entry in that list. Where it runs: the pre-push hook (`cargo xtask pre-push-check`) selects `xtask` -for any Rust change, the `windows-debug` job runs `cargo test --workspace` on -every pull request that changes Rust, and the merge queue and post-merge runs -test the whole workspace. The Linux pull-request lane (`cargo xtask affected`) -selects `xtask` only for a change in `xtask` or in a crate it depends on, or for -a file it reads, so a pull request that touches nothing but -`crates/rocm-dash-tui` does not run this test on Linux. The module-import -contract above has the same property. +for any Rust change; on every pull request that changes Rust, the +`windows-debug` job runs `cargo test --workspace` and the Linux `coverage` job +runs `cargo xtask coverage` (`cargo llvm-cov --workspace --exclude +e2e-cucumber`); and the merge queue and post-merge runs test the whole +workspace. The Linux `test` job (`cargo xtask affected`) selects `xtask` only +for a change in `xtask` or in a crate it depends on, or for a file it reads, so +a pull request that touches nothing but `crates/rocm-dash-tui` does not run this +test in that job, only in the `windows-debug` and `coverage` jobs. The +module-import contract above has the same property. ### Tests that run a fake executable @@ -628,8 +637,8 @@ when the diff of a changed Rust file adds or removes a line containing `proptest!`, so a property test moved inside `apps/rocm` alone still runs this check on the pull request, whether or not its seed moved with it. The other source-scan contracts do not have this rule; they still meet a change confined to -a crate that `xtask` does not depend on in the merge queue, which runs the whole -workspace. To prove a moved seed is read, +a crate that `xtask` does not depend on in the jobs that run the whole +workspace's tests (see [CI test selection](#ci-test-selection)). To prove a moved seed is read, append a deliberately unparsable line to it, run the test with `--nocapture` (`cargo test -p -- --nocapture`), look for proptest's `unparsable line, ignoring` warning naming that path, and remove the line. @@ -970,9 +979,54 @@ file, the workspace root manifest, CI config, or a doc an xtask test reads — the proptest seed guard and which no other crate depends on: a changed file under a `proptest-regressions` directory, and a changed Rust file whose diff adds or removes a line containing `proptest!` (found with `git diff -G`). On `main` -and in the merge queue the full workspace always runs. For a full local verification, keep using -`cargo test --workspace --all-targets` above — `affected` is a CI optimization, and it also -narrows the local pre-push hooks (next section). +and in the merge queue the full workspace always runs. For a full local verification, run +`cargo test --workspace --all-targets` above, which builds with the `e2e-test-hooks` feature +off, and the feature-on `cargo nextest run` commands just below — `affected` is a CI optimization, and it +also narrows the local pre-push hooks (next section). + +The `test` job's affected-crates `cargo nextest run` builds with the +`e2e-test-hooks` feature off, so the tests that exist only with it are not +among them. More steps of the `test` job +run them, each when its crate is in the selection or the selection is the +whole workspace. Each runs that crate's whole suite with the feature on (the +rocm-core one its library tests), not only the tests behind the feature: + +```bash +cargo nextest run --locked -p rocm-core --lib --features e2e-test-hooks +cargo nextest run --locked -p rocm --features rocm/e2e-test-hooks +``` + +Tests gated on the feature being off, such as +`runner_options_points_the_kfd_preflight_at_the_real_device` in +[dash.rs](../apps/rocm/src/dash.rs) and +`a_build_without_the_hook_ignores_the_variable` in +[hardware_root.rs](../crates/rocm-core/src/hardware_root.rs), are not built by +these steps; feature-off runs such as the plain `cargo nextest run` above +run them. + +A contract test in [workflow_contract.rs](../xtask/src/workflow_contract.rs), +`ci_lints_and_tests_every_crate_with_e2e_test_hooks_gated_tests`, finds every +crate whose library, binary, proc-macro or build-script module tree has test +code the feature changes: test code holding a `cfg(..)` or `cfg!(..)` the +feature can turn on or a `cfg_attr(..)` naming it, test code behind such a gate, +or a `mod x;` behind the feature, whose file it does not read. It fails while +such a crate has no step like these, so a crate that gains such test code needs +a step of its own. It does not read integration tests under a crate's `tests/` +directory, so a test there behind the feature is not checked. It also checks +that each such step has its exact `run:` and `if:` and no `continue-on-error`, +that the `clippy` job runs both workspace clippy commands, feature off and on, +and that every other crate with code behind the feature is in `rocm`'s +`e2e-test-hooks` feature list. Run it on its own with: + +```bash +cargo test -p xtask ci_lints_and_tests_every_crate_with_e2e_test_hooks_gated_tests +``` + +On a pull request the Linux `test` job runs it only when `xtask` is in the +affected selection, but the whole workspace's tests, this one included, run +whenever a Rust file changes: in the `windows-debug` job, and in the Linux +`coverage` job, whose `cargo xtask coverage` runs `cargo llvm-cov --workspace +--exclude e2e-cucumber`. ### Pre-push hook scope @@ -989,6 +1043,10 @@ pushed (`PRE_COMMIT_FROM_REF..PRE_COMMIT_TO_REF`, the commits the remote does no `e2e-cucumber` (its scenarios run through `cargo xtask e2e`), or `--workspace --all-targets --exclude e2e-cucumber` for the whole workspace. +The hooks build with `e2e-test-hooks` off, so the feature-on clippy and the feature-on +test steps run only in CI, or by hand with the feature-on commands under +[Local Verification](#local-verification) and [CI test selection](#ci-test-selection). + The directory-based selection cannot see a crate's tests reading a file from outside the crate, so the hook widens it: any Rust file also selects `xtask` (its contract tests scan every source file), `README.md`, `docs/` and `skills/` also select `rocm` and `xtask` (the advised-commands @@ -1014,7 +1072,8 @@ filter, as before, including a lockfile-only push. The hook is an early warning, and CI is not narrowed everywhere. On pull requests the `clippy` job lints the whole workspace and the `test` job (`Test (affected crates)`) runs only the crates `cargo xtask affected` selects. The whole workspace's tests run in the `windows-debug` job on -every change, and in the merge queue and on `main`, where the Linux `test` job is forced to +every Rust change, in the Linux `coverage` job (`cargo xtask coverage`, every crate but `e2e-cucumber`) +on every Rust change, and in the merge queue and on `main`, where the Linux `test` job is forced to `--workspace`. Known limits, unchanged by the narrowing: the steps check the working tree rather than the pushed @@ -4337,8 +4396,8 @@ as itself rather than silently treated as an unknown distribution. The two host root and check that both `rocm examine` reads answer from it through the shared parser: the first at `/etc/os-release`, the second with only `/usr/lib/os-release` present, so the fallback is exercised through the real -`host_path`-routed call sites, not just the helper in isolation; the feature -they need is enabled only by its own CI step. The +`host_path`-routed call sites, not just the helper in isolation; CI runs them +in the `test` job's rocm-core `e2e-test-hooks` step. The `try_os_release_does_not_source_the_fallback_when_the_primary_is_not_sourceable` test runs the WSL remote probe's own shell fallback for real under `/bin/sh` against a primary that exists but cannot be sourced (a directory in its @@ -4357,7 +4416,8 @@ and expects the real file back, and `system_packages::tests::the_install_hints_read_the_real_host_and_ignore_a_simulated_root` holds the three install hints to the same rule. Both live in that process rather than in `rocm-core`'s, so the environment variable they set cannot -overlap a `rocm-core` test that reads host paths. +overlap a `rocm-core` test that reads host paths. CI runs them in the `test` +job's rocm `e2e-test-hooks` step. `system_packages::tests::no_caller_in_this_binary_reads_the_real_os_release_by_hand` fails when a source file of the `rocm` binary reads the file with its own resolver instead of calling it. diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index cf12b2609..40e371fc6 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -38,4 +38,5 @@ tar = "0.4" zip = { version = "9.0", default-features = false, features = ["deflate"] } [dev-dependencies] +quote = "1" tempfile = "3" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e3db2768f..4186d227f 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -127,8 +127,10 @@ pub(crate) enum Command { /// /// The step runs over the whole workspace whenever the affected crates /// cannot be determined, and prints why. It is skipped only when the - /// selection is known and has nothing for the step to check. CI always - /// checks the whole workspace. + /// selection is known and has nothing for the step to check. On a pull + /// request with a Rust change, CI's Windows debug job tests the whole + /// workspace and its Linux coverage job every crate but `e2e-cucumber`; the + /// merge queue and `main` always test the whole workspace. /// /// A step that already passed on this exact tree (same cargo command, /// toolchain and environment, clean working tree) is skipped, and a pass is diff --git a/xtask/src/module_size.rs b/xtask/src/module_size.rs index 5b874c8c5..c8e5833ff 100644 --- a/xtask/src/module_size.rs +++ b/xtask/src/module_size.rs @@ -246,6 +246,72 @@ impl From<&MetaPackage> for Package { } } +/// A package name, and each source file its module tree reaches with whether it +/// is reached through at least one test gate. +#[cfg(test)] +pub(crate) type ReachedPackage = (String, Vec<(PathBuf, bool)>); + +/// Every `.rs` file each workspace package's targets reach (test-gated files, +/// and the files of a package under a `tests/` directory such as +/// `tests/e2e-cucumber`, included — unlike the size count), by package name, +/// with whether the file is reached through a test-gated `mod` declaration +/// (`#[cfg(test)] mod x;`, or any file below one), or through a file whose own +/// `#![cfg(test)]` gates it: the walk passes that gate on to the file's +/// children. A file reached both through a gate and without one is listed as +/// gated. The flag does not reflect a file's own `#![cfg(test)]` on that file +/// itself: a caller that needs it reads the file's inner attributes. It exists +/// for the `e2e-test-hooks` contract in `workflow_contract.rs`, which must +/// follow the module tree this check walks — including `#[path]` modules and +/// files one package loads from another's directory — rather than guess a +/// file's package and gating from where it sits. +/// +/// The walk starts only from library, binary, proc-macro and build-script +/// targets, so integration tests under a package's `tests/` directory are not +/// listed. +#[cfg(test)] +pub(crate) fn reached_source_files() -> Result> { + reached_source_files_in(&paths::run_cargo_metadata(&["--no-deps"])?) +} + +/// [`reached_source_files`] on one `cargo metadata --no-deps` document, so a +/// test can feed it a workspace of its own. +#[cfg(test)] +fn reached_source_files_in(stdout: &[u8]) -> Result> { + #[derive(Deserialize)] + struct NamedPackage { + name: String, + #[serde(flatten)] + package: MetaPackage, + } + #[derive(Deserialize)] + struct NamedMetadata { + workspace_root: PathBuf, + packages: Vec, + } + let metadata: NamedMetadata = + serde_json::from_slice(stdout).context("failed to parse `cargo metadata` output")?; + let mut reached = Vec::new(); + for named in &metadata.packages { + let mut walk = Walk { + root: normalize(&metadata.workspace_root), + counted: Vec::new(), + counted_files: BTreeSet::new(), + reached_files: BTreeSet::new(), + seen: BTreeSet::new(), + }; + for target_root in &Package::from(&named.package).roots { + walk.visit(target_root, true, false)?; + } + let mut files: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for (file, _, gated) in walk.seen { + *files.entry(file).or_default() |= gated; + } + reached.push((named.name.clone(), files.into_iter().collect())); + } + Ok(reached) +} + /// Production line count of every source file in `packages`, keyed by its path /// relative to `root` with `/` separators. fn collect(root: &Path, packages: &[Package]) -> Result> { @@ -599,7 +665,7 @@ fn path_attribute(attrs: &[Attribute]) -> Result> { } /// Whether any attribute makes an item exist only in test builds. -fn attrs_are_test_gated(attrs: &[Attribute]) -> Result { +pub(crate) fn attrs_are_test_gated(attrs: &[Attribute]) -> Result { for attr in attrs { let path = attr.path(); if path.is_ident("cfg") { @@ -670,7 +736,7 @@ fn cfg_is_test_only(predicate: &CfgTerm) -> Result { Ok(!terms.is_empty()) } -fn item_attrs(item: &Item) -> &[Attribute] { +pub(crate) fn item_attrs(item: &Item) -> &[Attribute] { match item { Item::Const(i) => &i.attrs, Item::Enum(i) => &i.attrs, @@ -691,7 +757,7 @@ fn item_attrs(item: &Item) -> &[Attribute] { } } -fn impl_item_attrs(item: &ImplItem) -> &[Attribute] { +pub(crate) fn impl_item_attrs(item: &ImplItem) -> &[Attribute] { match item { ImplItem::Const(i) => &i.attrs, ImplItem::Fn(i) => &i.attrs, @@ -701,7 +767,7 @@ fn impl_item_attrs(item: &ImplItem) -> &[Attribute] { } } -fn trait_item_attrs(item: &TraitItem) -> &[Attribute] { +pub(crate) fn trait_item_attrs(item: &TraitItem) -> &[Attribute] { match item { TraitItem::Const(i) => &i.attrs, TraitItem::Fn(i) => &i.attrs, @@ -1447,6 +1513,93 @@ trait T { ); } + /// The gated flag the `e2e-test-hooks` contract reads: true for a file + /// reached through a test-gated `mod` declaration and for the files below + /// it, false for an ordinary module, false for a file whose only gate is + /// its own `#![cfg(test)]`, which the flag does not reflect, true for a + /// file below that one, which inherits the gate, and true for a file + /// reached both through a gate and without one. `both_a.rs` and + /// `both_b.rs` differ in which declaration carries the gate: the plain + /// `mod` for one, the `#[path]` alias for the other. So a fold that keeps + /// either the first or the last set entry for a file gives `false` on one + /// of them. A second package, `user`, loads `demo/src/plain.rs` behind a + /// test gate through `#[path]`: each package gets its own list, under its + /// own name, so that file is gated for `user` and not for `demo`. + #[test] + fn reached_source_files_flag_files_reached_through_a_test_gated_mod() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + write( + root, + "Cargo.toml", + "[workspace]\nmembers = [\"demo\", \"user\"]\nresolver = \"2\"\n", + ); + write(root, "demo/Cargo.toml", &manifest("demo")); + write( + root, + "demo/src/lib.rs", + "mod plain;\nmod own;\n#[cfg(test)]\nmod gated;\n\ + #[cfg(test)]\nmod both_a;\n#[path = \"both_a.rs\"]\nmod both_a_alias;\n\ + mod both_b;\n#[cfg(test)]\n#[path = \"both_b.rs\"]\nmod both_b_alias;\n", + ); + write(root, "demo/src/both_a.rs", "fn a() {}\n"); + write(root, "demo/src/both_b.rs", "fn b() {}\n"); + write(root, "demo/src/plain.rs", "fn p() {}\n"); + write(root, "demo/src/own.rs", "#![cfg(test)]\nmod child;\n"); + write(root, "demo/src/own/child.rs", "fn c() {}\n"); + write(root, "demo/src/gated.rs", "mod below;\n"); + write(root, "demo/src/gated/below.rs", "fn b() {}\n"); + write(root, "user/Cargo.toml", &manifest("user")); + write( + root, + "user/src/lib.rs", + "#[cfg(test)]\n#[path = \"../../demo/src/plain.rs\"]\nmod borrowed;\n", + ); + + let mut reached = reached_source_files_in(&real_metadata(root)).unwrap(); + reached.sort_by(|a, b| a.0.cmp(&b.0)); + let root = normalize(root); + let flags: Vec<(&str, Vec<(String, bool)>)> = reached + .iter() + .map(|(name, files)| { + let files = files + .iter() + .map(|(file, gated)| { + let relative = file.strip_prefix(&root).unwrap_or(file); + (relative.to_string_lossy().replace('\\', "/"), *gated) + }) + .collect(); + (name.as_str(), files) + }) + .collect(); + let row = |file: &str, gated: bool| (file.to_string(), gated); + assert_eq!( + flags, + [ + ( + "demo", + vec![ + row("demo/src/both_a.rs", true), + row("demo/src/both_b.rs", true), + row("demo/src/gated/below.rs", true), + row("demo/src/gated.rs", true), + row("demo/src/lib.rs", false), + row("demo/src/own/child.rs", true), + row("demo/src/own.rs", false), + row("demo/src/plain.rs", false), + ] + ), + ( + "user", + vec![ + row("demo/src/plain.rs", true), + row("user/src/lib.rs", false), + ] + ), + ] + ); + } + #[test] fn evaluate_rejects_a_document_that_is_not_cargo_metadata() { let error = evaluate_metadata(b"not json").unwrap_err(); diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 9543c9211..1e5c10c9f 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -330,12 +330,15 @@ fn banner(check: Check, push: &Push, selection: &Selection, fallback: Option<&st checking the whole workspace", check.label() ), - // The claim about CI is backed by `.github/workflows/ci.yml`: the - // `windows-debug` job runs `cargo test --workspace --all-targets` on every - // change, and the `test` job selects `--workspace` for every event but + // The claim about CI rests on `.github/workflows/ci.yml`: the + // `windows-debug` job runs `cargo test --workspace --all-targets`, and + // the `test` job selects `--workspace` for every event but // `pull_request` (the merge queue and pushes to main). The test - // `the_banner_claim_about_ci_matches_the_workflow` fails if either stops - // being true, so change the text and that test together. + // `the_banner_claim_about_ci_matches_the_workflow` fails if either of + // those two stops being true, so change the text and that test + // together. Not checked by that test: the `windows-debug` step runs + // only when the `heavy` path filter matches, which every `.rs` file + // change does. None => format!( "pre-push-check {}: {} for the changes in {range}; CI's Windows job, merge queue \ and main still check the whole workspace", diff --git a/xtask/src/proptest_regressions_contract.rs b/xtask/src/proptest_regressions_contract.rs index 883d85eee..f65ea7184 100644 --- a/xtask/src/proptest_regressions_contract.rs +++ b/xtask/src/proptest_regressions_contract.rs @@ -44,8 +44,10 @@ //! never writes a seed file, and one with a different `FileFailurePersistence` //! would put its seeds outside `proptest-regressions/`, where this does not //! look. `cfg(test)` only, like the other source-scan contracts in this -//! crate: `cargo test -p xtask` runs it, and the merge queue (which runs the -//! whole workspace) is where a change confined to another crate meets it. +//! crate: `cargo test -p xtask` runs it, and a change confined to another crate +//! meets it in the jobs that run the whole workspace's tests: `windows-debug` +//! and, for every crate but `e2e-cucumber`, the Linux `coverage` job on every +//! Rust change, and the merge queue. #[cfg(test)] mod tests { diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index f0183849b..fe2f3b2b0 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2371,27 +2371,34 @@ jobs: Some(rest) => (rest.trim_start(), indent_of(line) + 2), None => (code, indent_of(line)), }; - let indicator = key.strip_prefix("run:")?.trim(); - let folded = match indicator.chars().next()? { - '|' => false, - '>' => true, - _ => return None, - }; - let modifiers = &indicator[1..]; - if !modifiers - .chars() - .all(|c| c == '-' || c == '+' || c.is_ascii_digit()) - { - return None; - } - let indent = modifiers - .chars() - .find_map(|c| c.to_digit(10)) - .map(|digit| digit as usize); + let (folded, indent) = block_scalar_header(key.strip_prefix("run:")?.trim())?; Some((at, key_column, folded, indent)) }) } + /// Whether `token` is a block-scalar header — `|` or `>`, followed only by + /// chomping (`-`/`+`) and indentation (digit) indicators — and if so, + /// whether it folds (`>`) and the indentation digit it names. + fn block_scalar_header(token: &str) -> Option<(bool, Option)> { + let folded = match token.chars().next()? { + '|' => false, + '>' => true, + _ => return None, + }; + let modifiers = &token[1..]; + if !modifiers + .chars() + .all(|c| c == '-' || c == '+' || c.is_ascii_digit()) + { + return None; + } + let indent = modifiers + .chars() + .find_map(|c| c.to_digit(10)) + .map(|digit| digit as usize); + Some((folded, indent)) + } + /// The script a step's block-scalar `run:` hands its shell, one entry per /// line: [`run_block`]'s lines, folded the way YAML folds a `>` block. /// @@ -5913,8 +5920,9 @@ fi } /// The value of the `{key}:` that sits exactly `indent` columns in, with any - /// continuation lines folded in and a leading block-scalar marker (`>-`, `|`) - /// dropped; `None` when there is no such key. Job-level keys sit at 4, and a + /// continuation lines folded in and a leading block-scalar header (as + /// [`block_scalar_header`] reads one) dropped; `None` when there is no such + /// key. Job-level keys sit at 4, and a /// step's own keys at the step's dash column plus 2. fn yaml_value_at(block: &str, indent: usize, key: &str) -> Option { let marker = format!("{key}:"); @@ -5937,13 +5945,42 @@ fi value.push(' '); value.push_str(code.trim()); } - Some( + // The header is the value's first word, which the folded first line + // follows after a space; a `|` or `>` with other text attached is not one. + let value = value.trim(); + let (head, rest) = value.split_once(' ').unwrap_or((value, "")); + let body = if block_scalar_header(head).is_some() { + rest + } else { value - .trim_start_matches(">-") - .trim_start_matches('|') - .trim() - .to_owned(), - ) + }; + Some(body.trim().to_owned()) + } + + /// `yaml_value_at` drops a block-scalar header of every form the step + /// readers accept, including one with nothing after it, and keeps a plain + /// value and a `|` or `>` with text attached, neither of which is a header. + /// `run_block_header` shares the recogniser and rejects a plain `run:`. + #[test] + fn yaml_value_at_drops_only_a_block_scalar_header() { + let value = |text: &str| { + yaml_value_at(&format!(" key: {text}\n"), 4, "key").expect("the key is there") + }; + for header in ["|", ">", "|-", ">-", "|+", ">+", "|2", ">2-", "|+2"] { + assert_eq!( + value(&format!("{header}\n a b\n c")), + "a b c", + "{header}" + ); + assert_eq!(value(header), "", "{header} with no lines after it"); + } + assert_eq!(value("plain text"), "plain text"); + assert_eq!(value("|cmd"), "|cmd"); + assert_eq!(value(">x y"), ">x y"); + assert_eq!(value("|2x\n a"), "|2x a"); + assert_eq!(value("x y"), "x y"); + assert_eq!(value("\n - a\n - b"), "- a - b"); + assert!(run_block_header(&[" - run: x"]).is_none()); } /// The aggregator's one step, as a step block. Panics unless it is the only @@ -10532,4 +10569,684 @@ permissions: "{log}" ); } + + /// Test code that only an `e2e-test-hooks` build compiles, and the CI + /// steps that must therefore lint it and run it with the feature on. + /// + /// No unit-test, nextest or clippy build in CI enables the feature except + /// the steps pinned here (the e2e lanes build the binaries with it, but no + /// unit tests), so a test behind it that no pinned step builds is not + /// compiled by any CI test or lint: it cannot fail, and the hook it covers + /// can break unnoticed. + mod e2e_test_hooks_coverage { + use std::collections::BTreeSet; + + use proc_macro2::{Delimiter, TokenStream, TokenTree}; + use quote::ToTokens; + use syn::{ImplItem, Item, TraitItem}; + + use super::{ + ci_job, indent_of, job_steps, read_workflow, repo_root, step_keys, yaml_value_at, + }; + use crate::module_size::{ + ReachedPackage, attrs_are_test_gated, impl_item_attrs, item_attrs, + reached_source_files, trait_item_attrs, + }; + + const FEATURE: &str = "e2e-test-hooks"; + + /// The `if:` every feature-on test step carries: Rust changed, and the + /// crate is in the affected selection or the whole workspace is. The + /// trailing space stops `-p rocm-core` from satisfying a check for + /// `-p rocm `. + fn selection_gate(krate: &str) -> String { + format!( + "needs.changes.outputs.rust == 'true' && (steps.sel.outputs.sel == '--workspace' \ + || contains(format('{{0}} ', steps.sel.outputs.sel), '-p {krate} '))" + ) + } + + /// Whether `tokens` hold a `feature = "e2e-test-hooks"` term, in + /// either polarity. + fn names_feature(tokens: &TokenStream) -> bool { + let (positive, negative) = feature_terms(tokens, false); + positive || negative + } + + /// The `feature = "e2e-test-hooks"` terms in `tokens`, a cfg predicate, + /// as `(positive, negative)`: positive when one sits under an even + /// number of `not(..)`s, so turning the feature on can make the + /// predicate hold, negative when one sits under an odd number. + /// `negated` says whether `tokens` are already under an odd number. + fn feature_terms(tokens: &TokenStream, negated: bool) -> (bool, bool) { + let trees: Vec = tokens.clone().into_iter().collect(); + let (mut positive, mut negative) = (false, false); + for (at, tree) in trees.iter().enumerate() { + match tree { + TokenTree::Ident(ident) if ident == "feature" => { + let is_ours = matches!( + (trees.get(at + 1), trees.get(at + 2)), + (Some(TokenTree::Punct(eq)), Some(TokenTree::Literal(lit))) + if eq.as_char() == '=' && lit.to_string() == format!("\"{FEATURE}\"") + ); + if is_ours { + if negated { + negative = true; + } else { + positive = true; + } + } + } + TokenTree::Group(group) => { + let flips = at > 0 + && matches!(&trees[at - 1], TokenTree::Ident(ident) if ident == "not"); + let (p, n) = feature_terms(&group.stream(), negated != flips); + positive |= p; + negative |= n; + } + _ => {} + } + } + (positive, negative) + } + + /// Whether `tokens` hold a cfg that the feature turns on: a + /// `cfg(..)` attribute or `cfg!(..)` whose predicate has a positive + /// feature term (see [`feature_terms`]), or any `cfg_attr(..)` naming + /// the feature, since either polarity there changes which attributes + /// (an `ignore`, say) the feature-on build sees. A `cfg(not(feature = + /// ..))` alone does not count: what it gates is built with the feature + /// off, which the plain steps already cover. + fn feature_turns_on_code(tokens: &TokenStream) -> bool { + let trees: Vec = tokens.clone().into_iter().collect(); + for (at, tree) in trees.iter().enumerate() { + let TokenTree::Group(group) = tree else { + continue; + }; + if group.delimiter() == Delimiter::Parenthesis && at > 0 { + let head = if matches!(&trees[at - 1], TokenTree::Punct(bang) if bang.as_char() == '!') + && at > 1 + { + &trees[at - 2] + } else { + &trees[at - 1] + }; + if let TokenTree::Ident(ident) = head { + if ident == "cfg" && feature_terms(&group.stream(), false).0 { + return true; + } + if ident == "cfg_attr" && names_feature(&group.stream()) { + return true; + } + } + } + if feature_turns_on_code(&group.stream()) { + return true; + } + } + false + } + + /// `node`'s tokens — for an item, its outer attributes included; for a + /// `syn::File`, its inner attributes and items — taken from the parsed + /// node itself rather than by re-lexing the source slice its span + /// covers, which breaks on CRLF line endings and a byte order mark. + fn tokens_of(node: &impl ToTokens) -> TokenStream { + node.to_token_stream() + } + + /// Whether `source` holds test code that only the feature-on build + /// compiles. `gated_file` is true when the file is reached through at + /// least one test gate (a `#[cfg(test)] mod x;` elsewhere), and then + /// everything in it is treated as test code: a feature-gated + /// production item in such a file is counted as hooks-only test code, + /// an over-report on the safe side. + /// + /// Test code is what `check-module-size` calls test-gated: an item + /// under `#[test]`, `#[tokio::test]` or a test-only `#[cfg(..)]`, and + /// everything inside one, macro bodies included. Inside it, any cfg the + /// feature turns on counts, down to a single statement. Test code + /// inside a non-test item that the feature turns on counts as well. + /// An external `mod x;` the feature turns on counts without its file + /// being read: the module walk records test gates only, so whether that + /// file holds tests is not known here, and it is assumed that it does. + fn has_hooks_only_test_code(source: &str, gated_file: bool) -> bool { + let file = syn::parse_file(source).expect("source parses"); + let gated = + gated_file || attrs_are_test_gated(&file.attrs).expect("file attributes parse"); + let in_feature = file + .attrs + .iter() + .any(|attr| feature_turns_on_code(&tokens_of(attr))); + items_have_hooks_only_test_code(&file.items, gated, in_feature) + } + + fn items_have_hooks_only_test_code(items: &[Item], gated: bool, in_feature: bool) -> bool { + items.iter().any(|item| { + let attrs = item_attrs(item); + let test = gated || attrs_are_test_gated(attrs).expect("attributes parse"); + if test { + return in_feature || feature_turns_on_code(&tokens_of(item)); + } + let in_feature = in_feature + || attrs + .iter() + .any(|attr| feature_turns_on_code(&tokens_of(attr))); + match item { + Item::Mod(module) => match &module.content { + Some((_, inner)) => { + items_have_hooks_only_test_code(inner, false, in_feature) + } + None => in_feature, + }, + Item::Impl(imp) => imp.items.iter().any(|member: &ImplItem| { + attrs_are_test_gated(impl_item_attrs(member)).expect("attributes parse") + && (in_feature || feature_turns_on_code(&tokens_of(member))) + }), + Item::Trait(tr) => tr.items.iter().any(|member: &TraitItem| { + attrs_are_test_gated(trait_item_attrs(member)).expect("attributes parse") + && (in_feature || feature_turns_on_code(&tokens_of(member))) + }), + _ => false, + } + }) + } + + /// Every workspace package with test code only the feature-on build + /// compiles, found by walking each package's module tree the way + /// `check-module-size` does. + fn crates_with_hooks_only_tests(packages: &[ReachedPackage]) -> BTreeSet { + packages + .iter() + .filter(|(_, files)| { + files.iter().any(|(file, gated)| { + let source = std::fs::read_to_string(file) + .unwrap_or_else(|e| panic!("reading {}: {e}", file.display())); + has_hooks_only_test_code(&source, *gated) + }) + }) + .map(|(name, _)| name.clone()) + .collect() + } + + /// Every workspace package whose sources hold a cfg naming the feature + /// in either polarity: its feature-on and feature-off builds compile + /// different code, so both have to be linted. + fn crates_with_hooks_seams(packages: &[ReachedPackage]) -> BTreeSet { + packages + .iter() + .filter(|(_, files)| { + files.iter().any(|(file, _)| { + let source = std::fs::read_to_string(file) + .unwrap_or_else(|e| panic!("reading {}: {e}", file.display())); + source_names_feature_in_a_cfg(&source) + }) + }) + .map(|(name, _)| name.clone()) + .collect() + } + + /// Whether the Rust file `source` holds a `cfg(..)`, `cfg!(..)` or + /// `cfg_attr(..)` naming the feature, in either polarity. It goes + /// through `syn::parse_file`, as the item scanner does, so the two + /// scanners accept the same input. Lexing the raw text would also drop + /// a byte order mark; what differs is a shebang line, which + /// `parse_file` drops and raw lexing fails on when it is not valid Rust + /// tokens. `parse_file` is also stricter (it needs valid items), which + /// costs nothing here: `reached_source_files` has already parsed every + /// listed file with it. + fn source_names_feature_in_a_cfg(source: &str) -> bool { + names_feature_in_a_cfg(&tokens_of(&syn::parse_file(source).expect("source parses"))) + } + + /// Whether `tokens` hold a `cfg(..)`, `cfg!(..)` or `cfg_attr(..)` + /// naming the feature, in either polarity. + fn names_feature_in_a_cfg(tokens: &TokenStream) -> bool { + let trees: Vec = tokens.clone().into_iter().collect(); + trees.iter().enumerate().any(|(at, tree)| { + let TokenTree::Group(group) = tree else { + return false; + }; + let head = trees[..at].iter().rev().find( + |t| !matches!(t, TokenTree::Punct(bang) if bang.as_char() == '!'), + ); + let is_cfg = group.delimiter() == Delimiter::Parenthesis + && matches!(head, Some(TokenTree::Ident(ident)) if ident == "cfg" || ident == "cfg_attr"); + (is_cfg && names_feature(&group.stream())) + || names_feature_in_a_cfg(&group.stream()) + }) + } + + /// The features `rocm`'s own `e2e-test-hooks` feature turns on in + /// other packages, as `package/feature` entries. + fn rocm_hooks_feature_entries() -> Vec { + let manifest = std::fs::read_to_string(repo_root().join("apps/rocm/Cargo.toml")) + .expect("apps/rocm/Cargo.toml is readable"); + let manifest: toml::Table = manifest.parse().expect("apps/rocm/Cargo.toml parses"); + manifest["features"][FEATURE] + .as_array() + .expect("rocm defines an `e2e-test-hooks` feature list") + .iter() + .map(|entry| { + entry + .as_str() + .expect("a feature entry is a string") + .to_owned() + }) + .collect() + } + + /// The command that runs `krate`'s tests with the feature on, matched + /// whole against a step's `run:` value, so an appended `|| true`, a + /// `--no-run`, a filter that selects nothing or a narrower target + /// fails the match. rocm-core's step names its own feature + /// unqualified and passes `--lib`, so rocm-core's integration tests + /// under `tests/` do not run with the feature. #510 added that step to + /// run rocm-core's lib tests with the feature. Every other crate takes + /// the plain form. + fn hooks_test_command(krate: &str) -> String { + match krate { + "rocm-core" => { + format!("cargo nextest run --locked -p rocm-core --lib --features {FEATURE}") + } + _ => format!("cargo nextest run --locked -p {krate} --features {krate}/{FEATURE}"), + } + } + + /// The clippy command that lints the workspace with the feature on. + fn hooks_clippy_command() -> String { + format!( + "cargo clippy --locked --workspace --all-targets --features rocm/{FEATURE} \ + -- -D warnings" + ) + } + + /// Whether `step`'s own `run:` value, comments removed and folded onto + /// one line, is exactly `command`. Only the `run:` key is read, so a + /// step whose name or comment quotes the command does not match, and a + /// multi-line script folds into something longer than one command. + fn runs_exactly(step: &str, command: &str) -> bool { + let item_indent = indent_of(step.lines().next().expect("a step has a first line")); + yaml_value_at(step, item_indent + 2, "run").as_deref() == Some(command) + } + + /// A step's own `if:`, folded onto one line. + fn step_if(step: &str) -> Option { + let item_indent = indent_of(step.lines().next().expect("a step has a first line")); + yaml_value_at(step, item_indent + 2, "if") + } + + /// Every crate with test code behind `e2e-test-hooks` has its tests + /// run with the feature on, and the clippy job lints the workspace + /// both with the feature off and with it on, reaching every crate that + /// has a seam. + /// + /// A test step's `run:` must be exactly [`hooks_test_command`] for the + /// crate, under exactly the selection gate the other affected-crate + /// steps use, and must not be made advisory. The two clippy commands + /// are matched whole against a step's `run:` the same way. + /// + /// The crates are found through each package's library, binary, + /// proc-macro and build-script module trees, the ones + /// `check-module-size` walks. + /// Integration tests under a crate's `tests/` directory are not read, + /// so a test there behind the feature is not checked. + #[test] + fn ci_lints_and_tests_every_crate_with_e2e_test_hooks_gated_tests() { + let packages = reached_source_files().expect("the workspace module tree can be walked"); + let crates = crates_with_hooks_only_tests(&packages); + for expected in ["rocm", "rocm-core"] { + assert!( + crates.contains(expected), + "`{expected}` is missing from the crates with tests behind `{FEATURE}`, \ + {crates:?}: either the scan has stopped seeing them (fix the scan) or the \ + crate's last such test was removed (drop it from this list)" + ); + } + + let ci = read_workflow("ci.yml"); + let test_steps = job_steps(&ci_job(&ci, "test")).expect("the test job has steps"); + for krate in &crates { + let command = hooks_test_command(krate); + let running: Vec<&(String, String)> = test_steps + .iter() + .filter(|(_, step)| runs_exactly(step, &command)) + .collect(); + assert!( + !running.is_empty(), + "`{krate}` has tests that exist only with `{FEATURE}`, so the `test` job \ + needs a step whose `run:` is exactly `{command}`; without one they are \ + compiled by the feature-on clippy step but never run" + ); + let gate = selection_gate(krate); + assert!( + running.iter().any(|(_, step)| { + step_if(step).as_deref() == Some(gate.as_str()) + && !step_keys(step).iter().any(|key| key == "continue-on-error") + }), + "a step running `{krate}`'s `{FEATURE}` tests must have exactly \ + `if: {gate}` (so it runs whenever `{krate}` is affected) and no \ + `continue-on-error`; found {:?}", + running.iter().map(|(name, _)| name).collect::>() + ); + } + + let clippy_steps = job_steps(&ci_job(&ci, "clippy")).expect("the clippy job has steps"); + for command in [ + "cargo clippy --locked --workspace --all-targets -- -D warnings".to_owned(), + hooks_clippy_command(), + ] { + assert!( + clippy_steps.iter().any(|(_, step)| { + runs_exactly(step, &command) + && step_if(step).as_deref() + == Some("needs.changes.outputs.rust == 'true'") + && !step_keys(step).iter().any(|key| key == "continue-on-error") + }), + "the clippy job must run `{command}` whenever Rust changed, with no \ + `continue-on-error`: the feature-off and feature-on runs each lint code the \ + other never compiles" + ); + } + + let seams = crates_with_hooks_seams(&packages); + for expected in ["rocm", "rocm-core", "rocm-engine-lemonade"] { + assert!( + seams.contains(expected), + "`{expected}` is missing from the crates with code behind `{FEATURE}`, \ + {seams:?}: either the seam scan has stopped seeing it (fix the scan) or the \ + crate's last such code was removed (drop it from this list)" + ); + } + let enabled = rocm_hooks_feature_entries(); + for krate in seams { + assert!( + krate == "rocm" || enabled.contains(&format!("{krate}/{FEATURE}")), + "`{krate}` has code behind `{FEATURE}`, but `rocm/{FEATURE}` does not turn \ + on `{krate}/{FEATURE}`, so the feature-on clippy step never lints it; add \ + `{krate}/{FEATURE}` to rocm's `{FEATURE}` feature list, or, if rocm does not \ + depend on `{krate}`, give the clippy job a feature-on run that reaches it and \ + teach this check to accept that run" + ); + } + } + + fn test_code(source: &str) -> bool { + has_hooks_only_test_code(source, false) + } + + #[test] + fn a_feature_gated_test_is_found() { + assert!(test_code( + "#[cfg(test)]\nmod tests {\n #[cfg(feature = \"e2e-test-hooks\")]\n #[test]\n fn t() {}\n}\n" + )); + assert!(test_code( + "#[cfg(all(feature = \"e2e-test-hooks\", target_os = \"linux\"))]\n#[tokio::test]\nasync fn t() {}\n" + )); + } + + #[test] + fn a_test_gated_on_the_feature_being_off_is_not_found() { + assert!(!test_code( + "#[cfg(test)]\nmod tests {\n #[cfg(not(feature = \"e2e-test-hooks\"))]\n #[test]\n fn t() {}\n}\n" + )); + assert!(!test_code( + "#[cfg(test)]\nmod tests {\n #[cfg(not(any(unix, feature = \"e2e-test-hooks\")))]\n #[test]\n fn t() {}\n}\n" + )); + } + + #[test] + fn a_doubly_negated_feature_counts_as_on() { + assert!(test_code( + "#[cfg(not(not(feature = \"e2e-test-hooks\")))]\n#[test]\nfn t() {}\n" + )); + } + + /// A gated module, a gated statement inside an ungated test, a + /// `cfg_attr` in either polarity, a `cfg!` and a gated helper inside + /// the test module all change what the feature-on build tests. + #[test] + fn gating_below_or_around_the_test_fn_is_found() { + for source in [ + "#[cfg(all(test, feature = \"e2e-test-hooks\"))]\nmod hooks {\n #[test]\n fn t() {}\n}\n", + "#[cfg(feature = \"e2e-test-hooks\")]\nmod hooks {\n #[cfg(test)]\n mod tests {\n #[test]\n fn t() {}\n }\n}\n", + "#[cfg(test)]\nmod tests {\n #[test]\n fn t() {\n #[cfg(feature = \"e2e-test-hooks\")]\n {\n assert!(true);\n }\n }\n}\n", + "#[cfg(test)]\nmod tests {\n #[cfg_attr(not(feature = \"e2e-test-hooks\"), ignore)]\n #[test]\n fn t() {}\n}\n", + "#[cfg(test)]\nmod tests {\n #[cfg_attr(feature = \"e2e-test-hooks\", ignore)]\n #[test]\n fn t() {}\n}\n", + "#[test]\nfn t() {\n if cfg!(feature = \"e2e-test-hooks\") {}\n}\n", + "#[cfg(test)]\nmod tests {\n #[cfg(feature = \"e2e-test-hooks\")]\n use std::env;\n}\n", + "#[cfg(feature = \"e2e-test-hooks\")]\nmod hooks;\n", + ] { + assert!(test_code(source), "not found in:\n{source}"); + } + } + + /// Each shape below reaches feature-on test code through one branch of + /// the scanner: a test-gated member of an impl or a trait that names + /// the feature itself, a plain test member of an impl or a trait the + /// feature gates, a file whose own `#![cfg(test)]` makes it test code, + /// and a file whose own `#![cfg(feature = ..)]` puts every test in it + /// behind the feature. + #[test] + fn impl_and_trait_members_and_inner_file_attributes_are_found() { + for (branch, source) in [ + ( + "impl member", + "struct S;\nimpl S {\n #[cfg(all(test, feature = \"e2e-test-hooks\"))]\n fn helper() {}\n}\n", + ), + ( + "trait member", + "trait T {\n #[cfg(all(test, feature = \"e2e-test-hooks\"))]\n fn helper() {}\n}\n", + ), + ( + "test member of a feature-gated impl", + "struct S;\n#[cfg(feature = \"e2e-test-hooks\")]\nimpl S {\n #[cfg(test)]\n fn helper() {}\n}\n", + ), + ( + "test member of a feature-gated trait", + "#[cfg(feature = \"e2e-test-hooks\")]\ntrait T {\n #[cfg(test)]\n fn helper() {}\n}\n", + ), + ( + "inner #![cfg(test)]", + "#![cfg(test)]\n#[cfg(feature = \"e2e-test-hooks\")]\npub fn plant() {}\n", + ), + ( + "inner feature attribute", + "#![cfg(feature = \"e2e-test-hooks\")]\n#[cfg(test)]\nmod tests {\n #[test]\n fn t() {}\n}\n", + ), + ] { + assert!(test_code(source), "{branch} not found in:\n{source}"); + } + } + + /// A Windows checkout gives `xtask/` and `tests/` CRLF line endings, + /// and a file can open with a byte order mark. Neither may make the + /// item scanner, [`has_hooks_only_test_code`], panic or change its + /// answer: a doc comment's span ends after the `\r` of its line + /// ending, and `syn::parse_file` strips a BOM before it records + /// offsets, so re-lexing a slice of the source by an item's span breaks + /// on both. + #[test] + fn crlf_line_endings_and_a_byte_order_mark_do_not_change_the_answer() { + let gated = "//! Inner docs.\n/// Outer docs.\nfn seam() {}\n#[cfg(test)]\nmod tests {\n \ + /// A test.\n #[cfg(feature = \"e2e-test-hooks\")]\n #[test]\n fn t() {}\n}\n"; + let plain = gated.replace("#[cfg(feature = \"e2e-test-hooks\")]", ""); + for (source, expected) in [(gated.to_owned(), true), (plain, false)] { + for variant in [ + source.replace('\n', "\r\n"), + format!("\u{feff}{source}"), + format!("\u{feff}{}", source.replace('\n', "\r\n")), + ] { + assert_eq!(test_code(&variant), expected, "{variant:?}"); + } + } + } + + /// Everything in a file only test builds reach is test code. + #[test] + fn a_gated_helper_in_a_test_only_file_is_found() { + let source = "#[cfg(feature = \"e2e-test-hooks\")]\npub fn plant() {}\n"; + assert!(has_hooks_only_test_code(source, true)); + assert!(!has_hooks_only_test_code(source, false)); + } + + /// Production seams, the feature named in a string or a comment, and + /// a different feature are not test code behind this one. + #[test] + fn production_seams_strings_and_other_features_are_not_found() { + for source in [ + "#[cfg(feature = \"e2e-test-hooks\")]\nfn seam() {}\n", + "#[cfg(test)]\nmod tests {\n #[test]\n fn t() {\n let _ = \"#[cfg(feature = \\\"e2e-test-hooks\\\")]\";\n }\n}\n", + "#[cfg(test)]\nmod tests {\n // #[cfg(feature = \"e2e-test-hooks\")]\n #[test]\n fn t() {}\n}\n", + "#[cfg(test)]\nmod tests {\n #[cfg(feature = \"other-hooks\")]\n #[test]\n fn t() {}\n}\n", + ] { + assert!(!test_code(source), "wrongly found in:\n{source}"); + } + } + + #[test] + fn the_seam_scan_sees_either_polarity_and_nothing_else() { + let seam = source_names_feature_in_a_cfg; + assert!(seam( + "#[cfg(not(feature = \"e2e-test-hooks\"))]\nfn f() {}\n" + )); + assert!(seam( + "fn f() -> bool { cfg!(feature = \"e2e-test-hooks\") }\n" + )); + assert!(seam( + "#[cfg_attr(feature = \"e2e-test-hooks\", allow(dead_code))]\nfn f() {}\n" + )); + assert!(seam("#![cfg(feature = \"e2e-test-hooks\")]\nfn f() {}\n")); + assert!(seam( + "#![cfg_attr(not(feature = \"e2e-test-hooks\"), allow(dead_code))]\nfn f() {}\n" + )); + assert!(!seam( + "fn f() -> &'static str { \"feature = e2e-test-hooks\" }\n" + )); + assert!(!seam( + "#[doc = \"cfg(feature = \\\"e2e-test-hooks\\\")\"]\nfn f() {}\n" + )); + } + + /// The seam scan reads a file with CRLF line endings, a byte order mark, + /// or a shebang line, and answers as it does for the plain text. Each + /// variant is written to a file and handed to [`crates_with_hooks_seams`], + /// so the test drives the same read-and-parse path the contract runs on + /// the workspace. The shebang rows hold an unterminated `"`, which is + /// not valid Rust tokens: they are the rows lexing the raw file would + /// fail on. The CRLF and byte-order-mark rows pin parity with the item + /// scanner, which reads the same variants. + #[test] + fn the_seam_scan_reads_crlf_bom_and_shebang_sources() { + let seam = + "//! Docs.\n/// Doc.\n#[cfg(not(feature = \"e2e-test-hooks\"))]\nfn f() {}\n"; + let plain = seam.replace("e2e-test-hooks", "other-hooks"); + let dir = tempfile::tempdir().expect("a scratch dir"); + for (source, expected) in [(seam.to_owned(), true), (plain, false)] { + for variant in [ + source.clone(), + source.replace('\n', "\r\n"), + format!("\u{feff}{source}"), + format!("\u{feff}{}", source.replace('\n', "\r\n")), + format!("#!/bin/sh -c \"x\n{source}"), + ] { + let file = dir.path().join("lib.rs"); + std::fs::write(&file, &variant).expect("write the fixture"); + let packages: Vec = + vec![("demo".to_owned(), vec![(file, false)])]; + assert_eq!( + crates_with_hooks_seams(&packages).contains("demo"), + expected, + "{variant:?}" + ); + } + } + } + + /// A step named after `command`, with `run` as its `run:` line. + fn step_named_for(command: &str, run: &str) -> String { + format!( + " - name: {command}\n # {command}\n if: x\n run: {run}\n" + ) + } + + /// The test step matcher takes only the exact command as the whole + /// `run:` value, and rejects every row below for not being that one + /// command. Some rows would keep the step green while its tests do not + /// run or a failure does not fail it (`|| true`, `--no-run`, an + /// `echo`); a `-E` filter narrows which tests run, and `--lib` changes + /// which targets are selected; the `set +e` script would still fail + /// with the command, since a script's status is its last command's, + /// but the check does not reason about scripts, so it accepts only the + /// command on its own. The command is still accepted with a trailing + /// comment, or alone under any block header. + #[test] + fn the_test_step_matcher_takes_only_the_exact_command() { + let rocm = hooks_test_command("rocm"); + assert_eq!( + rocm, + "cargo nextest run --locked -p rocm --features rocm/e2e-test-hooks" + ); + for run in [ + rocm.clone(), + format!("{rocm} # why"), + format!("|\n {rocm}"), + format!(">-\n {rocm}"), + format!("|-\n {rocm}"), + format!(">\n {rocm}"), + format!("|+\n {rocm}"), + format!("|2\n {rocm}"), + format!(">2-\n {rocm}"), + ] { + assert!( + runs_exactly(&step_named_for(&rocm, &run), &rocm), + "rejected `run: {run}`" + ); + } + for run in [ + format!("{rocm} || true"), + rocm.replace("run --locked", "run --no-run --locked"), + format!("{rocm} -E 'not test(/./)'"), + rocm.replace("-p rocm ", "-p rocm --lib "), + "echo skipped".to_owned(), + format!("echo {rocm}"), + format!("|\n set +e\n {rocm}"), + ] { + assert!( + !runs_exactly(&step_named_for(&rocm, &run), &rocm), + "accepted `run: {run}`" + ); + } + } + + /// The clippy commands go through the same matcher: an appended + /// `|| true`, an `echo` in place of the command, and a dropped + /// `--features` are each rejected. + #[test] + fn the_clippy_step_matcher_takes_only_the_exact_command() { + let clippy = hooks_clippy_command(); + assert!(runs_exactly(&step_named_for(&clippy, &clippy), &clippy)); + for run in [ + format!("{clippy} || true"), + "echo skipped".to_owned(), + clippy.replace(&format!(" --features rocm/{FEATURE}"), ""), + ] { + assert!( + !runs_exactly(&step_named_for(&clippy, &run), &clippy), + "accepted `run: {run}`" + ); + } + } + + #[test] + fn the_selection_gate_is_the_rocm_core_steps() { + let ci = read_workflow("ci.yml"); + let steps = job_steps(&ci_job(&ci, "test")).expect("the test job has steps"); + let (_, step) = steps + .iter() + .find(|(name, _)| name == "Test rocm-core with e2e-test-hooks") + .expect("the rocm-core step exists"); + assert_eq!(step_if(step), Some(selection_gate("rocm-core"))); + } + } }