a cache at capacity was weighing every resident key to evict one entry

examples/cache_scaling_bench.rs

@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 MatrixArkAI
+
+//! Read-path cost and read concurrency for the multi-tier cache.
+//!
+//! Two things are worth watching on the read path, and they move
+//! independently.
+//!
+//! The first is what a single memory-tier hit costs, and whether that cost
+//! stays put as the resident set grows. A lookup that scans anything will show
+//! up here as a number that climbs with the entry count.
+//!
+//! The second is what happens when several threads read at once. A cache that
+//! serialises its readers gets *slower* per operation as threads are added
+//! rather than faster in aggregate, and that shows up as throughput falling in
+//! the scaling table below. Reads currently take the cache lock exclusively
+//! because the hit updates statistics, hotness and latency histograms on the
+//! way out, so this table is the measurement to beat when that changes.
+//!
+//! The third table puts `MultiLayerCache` next to `ShardedMultiLayerCache`
+//! holding the same total capacity. The sharded cache spreads keys over
+//! independent shards, so readers of different shards do not queue behind one
+//! another. It is the supported answer to the contention in the second table,
+//! and this comparison is what says whether it earns its extra bookkeeping.
+//!
+//! ```text
+//! cargo run --release --example cache_scaling_bench
+//! cargo run --release --example cache_scaling_bench -- 8192
+//! ```
+
+use matrixcache::{CacheKey, CacheOptions, MultiLayerCache, ShardedMultiLayerCache};
+use std::time::{Duration, Instant};
+
+const VALUE_BYTES: usize = 64;
+const REPEATS: usize = 5;
+
+fn bench_dir(name: &str) -> std::path::PathBuf {
+    std::env::temp_dir().join(format!("matrixcache-scaling-{name}"))
+}
+
+/// Keys plus a visit order that touches each one but not in insertion order.
+fn workload(entries: usize) -> Vec<CacheKey> {
+    (0..entries)
+        .map(|index| CacheKey::string(0, &format!("scaling-key-{index:010}")))
+        .collect()
+}
+
+fn scattered(index: usize, len: usize) -> usize {
+    index.wrapping_mul(2_654_435_761) % len.max(1)
+}
+
+fn ns_per_op(elapsed: Duration, ops: usize) -> f64 {
+    if ops == 0 {
+        return 0.0;
+    }
+    elapsed.as_nanos() as f64 / ops as f64
+}
+
+/// Cost of a single-threaded memory-tier hit at a given resident entry count.
+fn hit_cost(entries: usize) -> f64 {
+    let dir = bench_dir(&format!("hit-{entries}"));
+    let _ = std::fs::remove_dir_all(&dir);
+    let cache = MultiLayerCache::new(entries * 256, &dir);
+    cache.start().expect("start cache");
+    let keys = workload(entries);
+    let value = vec![b'v'; VALUE_BYTES];
+    for key in &keys {
+        cache.put(key.clone(), value.clone()).expect("put");
+    }
+
+    let ops = entries * 8;
+    let mut best = f64::MAX;
+    for _ in 0..REPEATS {
+        let started = Instant::now();
+        for index in 0..ops {
+            let _ = cache.get(&keys[scattered(index, entries)]).expect("get");
+        }
+        best = best.min(ns_per_op(started.elapsed(), ops));
+    }
+    let _ = std::fs::remove_dir_all(&dir);
+    best
+}
+
+/// Aggregate read throughput with `threads` readers sharing one cache.
+fn read_throughput(entries: usize, threads: usize) -> f64 {
+    let dir = bench_dir(&format!("conc-{threads}"));
+    let _ = std::fs::remove_dir_all(&dir);
+    let cache = MultiLayerCache::new(entries * 256, &dir);
+    cache.start().expect("start cache");
+    let keys = workload(entries);
+    let value = vec![b'v'; VALUE_BYTES];
+    for key in &keys {
+        cache.put(key.clone(), value.clone()).expect("put");
+    }
+
+    let per_thread = 40_000usize;
+    let mut best = f64::MAX;
+    for _ in 0..3 {
+        let started = Instant::now();
+        std::thread::scope(|scope| {
+            for thread in 0..threads {
+                let cache = &cache;
+                let keys = &keys;
+                scope.spawn(move || {
+                    for index in 0..per_thread {
+                        // Offset per thread so readers are not in lockstep.
+                        let slot = scattered(index + thread * 7919, keys.len());
+                        let _ = cache.get(&keys[slot]).expect("get");
+                    }
+                });
+            }
+        });
+        best = best.min(ns_per_op(started.elapsed(), per_thread * threads));
+    }
+    let _ = std::fs::remove_dir_all(&dir);
+    best
+}
+
+/// Options shared by both cache shapes, so the comparison is like for like.
+fn scaling_options(dir: &std::path::Path, entries: usize) -> CacheOptions {
+    CacheOptions::new(entries * 256, 0, 0).with_ssd_paths(vec![dir.to_path_buf()])
+}
+
+/// Read throughput for the single-lock cache, built from shared options.
+fn single_lock_throughput(entries: usize, threads: usize) -> f64 {
+    let dir = bench_dir(&format!("single-{threads}"));
+    let _ = std::fs::remove_dir_all(&dir);
+    let cache = MultiLayerCache::with_options(scaling_options(&dir, entries));
+    cache.start().expect("start cache");
+    let keys = workload(entries);
+    let value = vec![b'v'; VALUE_BYTES];
+    for key in &keys {
+        cache.put(key.clone(), value.clone()).expect("put");
+    }
+    let best = drive_readers(threads, &keys, |key| {
+        let _ = cache.get(key).expect("get");
+    });
+    let _ = std::fs::remove_dir_all(&dir);
+    best
+}
+
+/// Read throughput for the sharded cache holding the same total capacity.
+fn sharded_throughput(entries: usize, threads: usize, shards: usize) -> f64 {
+    let dir = bench_dir(&format!("sharded-{shards}-{threads}"));
+    let _ = std::fs::remove_dir_all(&dir);
+    let cache = ShardedMultiLayerCache::with_options(scaling_options(&dir, entries), shards);
+    cache.start().expect("start cache");
+    let keys = workload(entries);
+    let value = vec![b'v'; VALUE_BYTES];
+    for key in &keys {
+        cache.put(key.clone(), value.clone()).expect("put");
+    }
+    let best = drive_readers(threads, &keys, |key| {
+        let _ = cache.get(key).expect("get");
+    });
+    let _ = std::fs::remove_dir_all(&dir);
+    best
+}
+
+/// Run `threads` readers over `keys` and return the best ns/op of three runs.
+fn drive_readers<F>(threads: usize, keys: &[CacheKey], read: F) -> f64
+where
+    F: Fn(&CacheKey) + Sync,
+{
+    let per_thread = 40_000usize;
+    let mut best = f64::MAX;
+    for _ in 0..3 {
+        let started = Instant::now();
+        std::thread::scope(|scope| {
+            for thread in 0..threads {
+                let read = &read;
+                scope.spawn(move || {
+                    for index in 0..per_thread {
+                        // Offset per thread so readers are not in lockstep.
+                        let slot = scattered(index + thread * 7919, keys.len());
+                        read(&keys[slot]);
+                    }
+                });
+            }
+        });
+        best = best.min(ns_per_op(started.elapsed(), per_thread * threads));
+    }
+    best
+}
+
+fn main() {
+    let max_entries: usize = std::env::args()
+        .nth(1)
+        .and_then(|value| value.parse().ok())
+        .unwrap_or(4_096);
+
+    // Warm the allocator before the first measured case.
+    let _ = hit_cost(256);
+
+    println!("memory-tier hit, single thread");
+    println!("{:>10} {:>14}", "entries", "ns/op");
+    let mut size = 1_024usize;
+    while size <= max_entries {
+        println!("{size:>10} {:>14.1}", hit_cost(size));
+        size *= 4;
+    }
+
+    println!();
+    println!("read throughput, {max_entries} resident entries");
+    println!("{:>10} {:>14} {:>14}", "threads", "ns/op", "Mops/s");
+    for &threads in &[1usize, 2, 4, 8] {
+        let ns = read_throughput(max_entries, threads);
+        let mops = if ns > 0.0 { 1_000.0 / ns } else { 0.0 };
+        println!("{threads:>10} {ns:>14.1} {mops:>14.2}");
+    }
+
+    println!();
+    println!("single lock vs sharded, same total capacity, Mops/s");
+    println!(
+        "{:>10} {:>14} {:>14} {:>10}",
+        "threads", "single", "sharded", "speedup"
+    );
+    for &threads in &[1usize, 2, 4, 8] {
+        let single_ns = single_lock_throughput(max_entries, threads);
+        let sharded_ns = sharded_throughput(max_entries, threads, 16);
+        let single = if single_ns > 0.0 {
+            1_000.0 / single_ns
+        } else {
+            0.0
+        };
+        let sharded = if sharded_ns > 0.0 {
+            1_000.0 / sharded_ns
+        } else {
+            0.0
+        };
+        let speedup = if single > 0.0 { sharded / single } else { 0.0 };
+        println!("{threads:>10} {single:>14.2} {sharded:>14.2} {speedup:>9.2}x");
+    }
+}

examples/eviction_bench.rs

@@ -0,0 +1,152 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 MatrixArkAI
+
+//! Cost of choosing an eviction victim, as the resident set grows.
+//!
+//! A cache at capacity evicts on almost every write, so whatever victim
+//! selection costs is paid per write for the life of the cache. The thing to
+//! watch is whether that cost stays put as the cache fills: a selector that
+//! inspects every resident entry shows up here as a per-write cost that climbs
+//! with the entry count, while one that inspects a bounded number of
+//! candidates shows up as a flat line.
+//!
+//! Two numbers are reported per size. The first is wall time per write, which
+//! is what a caller feels. The second is the number of candidate groups the
+//! selector formed per evicted entry, which the cache already counts; it is
+//! immune to load on the machine and is the number that says whether the
+//! algorithm changed or only the weather did.
+//!
+//! ```text
+//! cargo run --release --no-default-features --example eviction_bench
+//! ```
+
+use matrixcache::{CacheKey, CacheOptions, MultiLayerCache};
+use std::time::Instant;
+
+const VALUE_BYTES: usize = 64;
+/// Room for the value plus its per-entry overhead, so `entries` really fit.
+const SLOT_BYTES: usize = VALUE_BYTES;
+
+fn bench_dir(name: &str) -> std::path::PathBuf {
+    std::env::temp_dir().join(format!("matrixcache-eviction-{name}"))
+}
+
+fn key(index: usize) -> CacheKey {
+    CacheKey::string(0, &format!("eviction-key-{index:010}"))
+}
+
+/// Spread successive steps over `len` slots so reads are not in insertion
+/// order, which would flatter any policy that evicts from one end.
+fn scattered(index: usize, len: usize) -> usize {
+    index.wrapping_mul(2_654_435_761) % len.max(1)
+}
+
+/// Fill to capacity, then keep writing so every write evicts.
+fn steady_state(entries: usize) -> (f64, f64) {
+    let dir = bench_dir(&format!("steady-{entries}"));
+    let _ = std::fs::remove_dir_all(&dir);
+    let cache = MultiLayerCache::with_options(
+        CacheOptions::new(entries * SLOT_BYTES, 0, 0).with_ssd_paths(vec![dir.clone()]),
+    );
+    cache.start().expect("start cache");
+
+    let value = vec![b'v'; VALUE_BYTES];
+    // Fill to capacity. Keys are built up front so the timed region below
+    // measures the cache rather than key formatting.
+    for index in 0..entries {
+        cache.put(key(index), value.clone()).expect("put");
+    }
+
+    let writes = 2_000usize;
+    let fresh: Vec<CacheKey> = (entries..entries + writes).map(key).collect();
+
+    let before = cache.stats();
+    let started = Instant::now();
+    for k in &fresh {
+        cache.put(k.clone(), value.clone()).expect("put");
+    }
+    let elapsed = started.elapsed();
+    let after = cache.stats();
+
+    let evictions = after
+        .memory_evictions
+        .saturating_sub(before.memory_evictions);
+    let groups = after
+        .eviction_sampled_groups
+        .saturating_sub(before.eviction_sampled_groups);
+
+    let ns_per_write = elapsed.as_nanos() as f64 / writes as f64;
+    let groups_per_eviction = if evictions == 0 {
+        0.0
+    } else {
+        groups as f64 / evictions as f64
+    };
+
+    cache.stop();
+    let _ = std::fs::remove_dir_all(&dir);
+    (ns_per_write, groups_per_eviction)
+}
+
+/// Hit rate under a skewed read-through workload.
+///
+/// Bounding the candidate search only pays off if it still throws out the
+/// right entries. This drives a working set several times larger than the
+/// cache, with most reads landing on a small hot subset, and reports the share
+/// of reads the cache served. A selector that evicts hot entries shows up here
+/// as a hit rate below what the hot subset alone would guarantee.
+fn hit_rate(entries: usize) -> f64 {
+    let dir = bench_dir(&format!("hitrate-{entries}"));
+    let _ = std::fs::remove_dir_all(&dir);
+    let cache = MultiLayerCache::with_options(
+        CacheOptions::new(entries * SLOT_BYTES, 0, 0).with_ssd_paths(vec![dir.clone()]),
+    );
+    cache.start().expect("start cache");
+
+    let value = vec![b'v'; VALUE_BYTES];
+    // Four times as many keys as fit, with a hot subset that is half the
+    // cache, so a selector that protects hot entries can hold all of them.
+    let universe = entries * 4;
+    let hot = entries / 2;
+    let keys: Vec<CacheKey> = (0..universe).map(key).collect();
+
+    let reads = 400_000usize;
+    let mut hits = 0usize;
+    for step in 0..reads {
+        // Four reads in five land in the hot subset; the rest sweep the
+        // universe and are the pressure that forces eviction.
+        let slot = if step % 5 < 4 {
+            scattered(step, hot)
+        } else {
+            scattered(step, universe)
+        };
+        let k = &keys[slot];
+        if cache.get(k).expect("get").is_some() {
+            hits += 1;
+        } else {
+            cache.put(k.clone(), value.clone()).expect("put");
+        }
+    }
+
+    cache.stop();
+    let _ = std::fs::remove_dir_all(&dir);
+    hits as f64 * 100.0 / reads as f64
+}
+
+fn main() {
+    println!("steady-state write cost with the cache at capacity");
+    println!(
+        "{:>10}  {:>14}  {:>22}",
+        "entries", "ns/write", "groups/eviction"
+    );
+    for entries in [1_024usize, 2_048, 4_096, 8_192, 16_384, 32_768] {
+        let (ns, groups) = steady_state(entries);
+        println!("{entries:>10}  {ns:>14.0}  {groups:>22.1}");
+    }
+
+    println!();
+    println!("hit rate, working set 4x the cache, 80% of reads on a hot half-cache");
+    println!("{:>10}  {:>14}", "entries", "hit rate %");
+    for entries in [1_024usize, 4_096, 16_384] {
+        println!("{entries:>10}  {:>14.2}", hit_rate(entries));
+    }
+}

examples/policy_bench.rs

@@ -5,7 +5,7 @@
 //!
 //! Every policy keeps its keys in intrusive doubly-linked lists over a shared
 //! node arena, so a lookup or a delete unlinks an entry in constant time rather
-//! than rescanning a list of keys. The segmented LRU additionally partitions
+//! than rescanning a list of keys. The segmented Lru additionally partitions
 //! its index and lists into segments, each with its own byte budget, so
 //! eviction only ever walks a segment-local list.
 //!