Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,9 @@ jobs:
fail_ci_if_error: false

# P2-T010: loom models of the memory pool accounting, spill protocol, round
# admission and arbitration (`crates/krishiv-memory/src/loom_tests.rs`);
# admission and arbitration (`crates/krishiv-memory/src/loom_model.rs`);
# P2-T050: the executor's morsel cursor, finalize countdown, cancellation
# and result channel (`crates/krishiv-exec/src/loom_tests.rs`).
# and result channel (`crates/krishiv-exec/src/loom_model.rs`).
# P2-T163: runtime filter snapshot publication
# (`crates/krishiv-io/src/runtime_filter/tests.rs`).
# P2-T103: every wheel target, built and tested (wheels.yml).
Expand All @@ -181,6 +181,9 @@ jobs:
env:
RUSTC_WRAPPER: "" # `--cfg loom` rebuilds every dependency; not worth caching
RUSTFLAGS: --cfg loom
# Each step selects only the `loom_model` modules: a loom primitive used
# outside `loom::model` panics, so ordinary tests must never be selected
# (xtask policy test `loom_steps_select_only_loom_models`).
steps:
- name: Harden the runner (audit egress)
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
Expand All @@ -192,11 +195,11 @@ jobs:
- name: Toolchain (from rust-toolchain.toml)
run: rustup show active-toolchain
- name: loom models (krishiv-memory)
run: cargo test -p krishiv-memory --lib --release --locked
run: "cargo test -p krishiv-memory --lib --release --locked loom_model::"
- name: loom models (krishiv-exec)
run: cargo test -p krishiv-exec --lib --release --locked
run: "cargo test -p krishiv-exec --lib --release --locked loom_model::"
- name: loom models (krishiv-io runtime filters)
run: cargo test -p krishiv-io --lib --release --locked runtime_filter
run: "cargo test -p krishiv-io --lib --release --locked loom_model::"

# cargo-fuzz targets (`fuzz/`), one hour each: the reusable `fuzz.yml`
# (regression replay, cached corpus, crash triage). The weekly long run is
Expand Down
4 changes: 2 additions & 2 deletions crates/krishiv-exec/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ Batch execution engine: operators, pipelines, morsel scheduling within a node.
driver (yields, batch-boundary checks, NeedMemory, spill servicing, stall detector) · `runner.rs` QueryRunner /
Scheduler (coordinator, first error wins) · `channel.rs` bounded result stream · `executor.rs` Executor seam +
TokioExecutor · `context.rs` QueryContext/TaskContext/MemoryHandle · `morsel.rs` cursor + finalize countdown ·
`testing.rs` test operators (also used by benches) · `output_spill.rs` spill hook of finished breaker output · `loom_tests.rs`.
`testing.rs` test operators (also used by benches) · `output_spill.rs` spill hook of finished breaker output · `loom_model.rs`.
`operators/` (P2-T051) stateless operators: `scan.rs` MemoryScan + `ScanPushdown` (pushdowns decided once as an
immutable `ScanPushdowns`; implemented by scan factories: MemoryScan, FileScan) ·
`filter.rs` · `project.rs` · `limit.rs` · `union.rs` (per-branch `UnionAlign`; UNION ALL fan-in is T074, distinct UNION is the aggregate's) ·
Expand Down Expand Up @@ -84,5 +84,5 @@ Paths are relative to this crate.

## Tests
`cargo nextest run -p krishiv-exec --no-tests=pass` · `cargo clippy -p krishiv-exec --all-targets -- -D warnings` ·
loom models: `RUSTFLAGS="--cfg loom" cargo test -p krishiv-exec --lib --release` (use a separate target dir) ·
loom models: `RUSTFLAGS="--cfg loom" cargo test -p krishiv-exec --lib --release loom_model::` (use a separate target dir) ·
benches: `cargo bench -p krishiv-exec --bench pipeline` · `--bench operators`
4 changes: 2 additions & 2 deletions crates/krishiv-exec/src/join/grace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1343,9 +1343,9 @@ fn scatter(

/// Loom models of the grace phase's claims and waits (no I/O: tasks with
/// no rows). `RUSTFLAGS="--cfg loom" cargo test -p krishiv-exec --lib
/// --release`.
/// --release loom_model::`.
#[cfg(all(test, loom))]
mod loom_tests {
mod loom_model {
use loom::future::block_on;
use loom::thread;

Expand Down
2 changes: 1 addition & 1 deletion crates/krishiv-exec/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -127,4 +127,4 @@ pub use runner::{QueryRunner, Scheduler};
mod tests;

#[cfg(all(test, loom))]
mod loom_tests;
mod loom_model;
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
//! protocol model was checked to fail with a planted bug (Relaxed
//! countdown; check-before-listen in the wait loop, in `blocking_done` and
//! in `Barrier::wait`; `drop_receiver` not waking senders).
//! `RUSTFLAGS="--cfg loom" cargo test -p krishiv-exec --lib --release`.
//! `RUSTFLAGS="--cfg loom" cargo test -p krishiv-exec --lib --release loom_model::`.

use std::sync::Arc;

Expand Down
2 changes: 1 addition & 1 deletion crates/krishiv-exec/src/sync.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
//! Synchronisation primitives, swapped for loom's under `--cfg loom` so the
//! models in `loom_tests.rs` explore every interleaving of the morsel
//! models in `loom_model.rs` explore every interleaving of the morsel
//! cursor, the finalize countdown, cancellation and the result channel.
//!
//! [`Event`] is a broadcast wake-up with no lost notifications: a listener
Expand Down
2 changes: 1 addition & 1 deletion crates/krishiv-io/src/runtime_filter/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -648,7 +648,7 @@ mod unit {
/// The publication protocol under loom: a reader that sees a generation
/// finds that generation's snapshot (never an older one), and a consumer's
/// cache never pairs a generation with another snapshot.
/// `RUSTFLAGS="--cfg loom" cargo test -p krishiv-io --lib --release runtime_filter`.
/// `RUSTFLAGS="--cfg loom" cargo test -p krishiv-io --lib --release loom_model::`.
#[cfg(loom)]
mod loom_model {
use std::sync::Arc;
Expand Down
6 changes: 3 additions & 3 deletions crates/krishiv-memory/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Paths are relative to this crate.
- `src/spill.rs` spill scopes and spill accounting
- `src/spillfile/format.rs` on-disk spill format; `src/spillfile/writer.rs` and `reader.rs` frame I/O
- `src/spillfile/dir.rs` spill directory, advisory lock and crash reclaim
- `src/loom_tests.rs` loom models; `src/sync.rs` loom/std primitive switch
- `src/loom_model.rs` loom models; `src/sync.rs` loom/std primitive switch

## Extension points
- New memory consumer: take a `Reservation` from the pool and grow it with `try_grow`; do not allocate unaccounted.
Expand All @@ -23,7 +23,7 @@ Paths are relative to this crate.
## Invariants
- Every operator allocation is accounted through reservations; pools are empty at query end (tested).
- Arbitration and OOM paths return typed errors or trigger spill; never panic.
- Concurrency primitives tested with loom (`src/loom_tests.rs`; `src/sync.rs` swaps in loom's types).
- Concurrency primitives tested with loom (`src/loom_model.rs`; `src/sync.rs` swaps in loom's types).
- Arbitration is cooperative: never call into, lock or mutate another task's operator state; no sync lock across an await.
- `unsafe` allowed; every block has a `// SAFETY:` comment; runs under Miri in CI and nightly ASan/TSan (ADR 0029).
- Spill files (`spillfile`): a finished `.spill` is complete (temp file + rename; no fsync by design: scratch data, deleted at the
Expand All @@ -37,4 +37,4 @@ Paths are relative to this crate.

## Tests
`cargo nextest run -p krishiv-memory --no-tests=pass` · `cargo clippy -p krishiv-memory --all-targets -- -D warnings` ·
loom models: `RUSTFLAGS="--cfg loom" cargo test -p krishiv-memory --lib --release` (use a separate target dir)
loom models: `RUSTFLAGS="--cfg loom" cargo test -p krishiv-memory --lib --release loom_model::` (use a separate target dir)
2 changes: 1 addition & 1 deletion crates/krishiv-memory/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,4 +208,4 @@ pub trait MemoryPool: Send + Sync + std::fmt::Debug {
mod tests;

#[cfg(all(test, loom))]
mod loom_tests;
mod loom_model;
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
//! Loom models of reserve/release accounting and the cooperative spill
//! protocol (request, ack, give-up). Bounded for CI:
//! `RUSTFLAGS="--cfg loom" cargo test -p krishiv-memory --lib --release`.
//! `RUSTFLAGS="--cfg loom" cargo test -p krishiv-memory --lib --release loom_model::`.

use std::sync::Arc;
use std::time::Duration;
Expand Down
2 changes: 1 addition & 1 deletion crates/krishiv-memory/src/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//! Synchronisation primitives, swapped for loom's under `--cfg loom`, so
//! the model tests explore every interleaving of the accounting, wake-up
//! and arbitration code (the models inject a clock that never fires)
//! (`RUSTFLAGS="--cfg loom" cargo test -p krishiv-memory --lib --release`).
//! (`RUSTFLAGS="--cfg loom" cargo test -p krishiv-memory --lib --release loom_model::`).

#[cfg(all(test, loom))]
pub(crate) use loom::sync::{
Expand Down
105 changes: 105 additions & 0 deletions xtask/tests/policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -784,3 +784,108 @@ fn release_publishing_is_manual_and_gated() {
"quickstart-test.sh must run Python with TMPDIR=\"$work\""
);
}

/// The `cargo test` commands of a loom job that do not select exactly the
/// `loom_model` modules (`--lib`, filter `loom_model::`). The `run:` value
/// is YAML-quoted (a trailing `::` is not a valid plain scalar), so
/// surrounding quotes are stripped before matching.
fn loom_steps_off_the_models(loom_job: &str) -> Vec<String> {
loom_job
.lines()
.map(|l| l.trim().trim_start_matches("- "))
.filter_map(|l| l.strip_prefix("run: "))
.map(|cmd| {
cmd.trim()
.trim_matches(|c| c == '"' || c == '\'')
.trim()
.to_owned()
})
.filter(|cmd| cmd.starts_with("cargo test"))
.filter(|cmd| {
let words: Vec<&str> = cmd.split_whitespace().collect();
let last = words
.last()
.map(|w| w.trim_matches(|c| c == '"' || c == '\''));
!(words.contains(&"--lib") && last == Some("loom_model::"))
})
.collect()
}

/// The crates whose sources build a `loom::model`, each checked to keep its
/// models in a module named `loom_model`.
fn crates_with_loom_models() -> Vec<String> {
fn walk(dir: &Path, files: &mut Vec<PathBuf>) {
for entry in std::fs::read_dir(dir).unwrap() {
let path = entry.unwrap().path();
if path.is_dir() {
walk(&path, files);
} else if path.extension().is_some_and(|e| e == "rs") {
files.push(path);
}
}
}
let mut crates = Vec::new();
for entry in std::fs::read_dir(root().join("crates")).unwrap() {
let krate = entry.unwrap().path();
let src = krate.join("src");
if !src.is_dir() {
continue;
}
let mut files = Vec::new();
walk(&src, &mut files);
for file in files {
let text = std::fs::read_to_string(&file).unwrap();
if !text.contains("loom::model") {
continue;
}
assert!(
file.file_stem().is_some_and(|s| s == "loom_model")
|| text.contains("mod loom_model {"),
"{} builds a loom model outside a `loom_model` module",
file.display()
);
let name = krate.file_name().unwrap().to_string_lossy().into_owned();
if !crates.contains(&name) {
crates.push(name);
}
}
}
crates.sort();
crates
}

/// A loom primitive used outside `loom::model` panics, so the nightly loom
/// steps select only the `loom_model` modules, and every crate with loom
/// models has a step.
#[test]
fn loom_steps_select_only_loom_models() {
let nightly = read(".github/workflows/nightly.yml");
let loom = job(&nightly, "loom");
assert_eq!(loom_steps_off_the_models(loom), Vec::<String>::new());
let crates = crates_with_loom_models();
assert!(!crates.is_empty(), "no loom models found");
for krate in crates {
let step = format!("run: \"cargo test -p {krate} --lib --release --locked loom_model::\"");
assert!(loom.contains(&step), "the nightly loom job lacks {step:?}");
}
}

#[test]
fn loom_step_check_rejects_name_filters_and_whole_crates() {
let job = " steps:\n - run: rustup show\n \
- run: cargo test -p a --lib --release --locked runtime_filter\n \
- run: cargo test -p b --lib --release --locked\n \
- run: cargo test -p c --release --locked loom_model::\n \
- run: cargo test -p d --lib --release --locked loom_model::\n \
- run: \"cargo test -p e --lib --release --locked loom_model::\"\n \
- run: 'cargo test -p f --lib --release --locked loom_model::'\n \
- run: cargo test -p g --lib --release --locked \"loom_model::\"\n";
assert_eq!(
loom_steps_off_the_models(job),
[
"cargo test -p a --lib --release --locked runtime_filter",
"cargo test -p b --lib --release --locked",
"cargo test -p c --release --locked loom_model::",
]
);
}
Loading