From 7175007e3bcaa3ce27a125fd2faa7c9688f075f5 Mon Sep 17 00:00:00 2001 From: Deeproute Date: Sun, 23 Aug 2026 17:58:45 -0700 Subject: [PATCH] Bring the published tree up to the source The published snapshot dates from 2026-08-12; the source has moved on by about 3,350 lines across 10 files. The largest piece is the replacement policy work -- including finding the eviction victim from the cold end rather than scanning -- along with cache facades, the multilayer cache, storage engines, allocator and GC wiring, and a policy_bench example that was not published at all. The test module grows by roughly 1,000 lines with it, so the behaviour arrives with the tests that pin it rather than ahead of them. `cargo test` is green: 277 passed. --- CHANGELOG.md | 157 ++++ examples/policy_bench.rs | 408 +++++++++ src/core/storage_config.rs | 31 + src/runtime/allocators_executors.rs | 47 ++ src/runtime/builder_and_gc.rs | 114 +-- src/runtime/cache_facades.rs | 517 +++++++++++- src/runtime/multilayer_cache.rs | 143 ++-- src/runtime/replacement.rs | 1218 +++++++++++++++++++++++---- src/runtime/storage_engines.rs | 72 +- src/tests/mod.rs | 1033 ++++++++++++++++++++++- 10 files changed, 3353 insertions(+), 387 deletions(-) create mode 100755 examples/policy_bench.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7777944..5a85c09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,80 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed +- **`L2CachePolicy` buffers access records again instead of applying them on + the calling path.** `set_async_on_access` picks between the two modes. + Buffering keeps the caller off the migration-order update path at the cost of + that order lagging the workload, and a record arriving once the buffer is + full is dropped and counted by `access_drop_count`. Previously every record + was enqueued and drained in the same call, so the buffer bound was + unreachable and the shedding behaviour could never trigger. +- **Queueing an evicted buffer for the lower tier is opt-in.** + `set_use_eviction_handler` controls it and it is off by default, so an + eviction drops the data and the tail passes stay the only path into the lower + tier. It previously queued unconditionally, which wrote every eviction down a + tier whether or not that was wanted. +- **A failed migration write no longer stalls the queue behind it.** + `L2CachePolicy::write_task_internal` counts the failure and keeps draining + rather than returning early on the first error, so one bad key cannot block + every entry queued after it. Failures remain visible through + `write_fail_count`. +- **`L2CachePolicyFactory` sizes from the documented defaults** — 100,000 + migration-order items, 100,000 buffered access records, 1,000 keys per tail + batch and 10,000 queued writes — rather than the placeholder 1,024 / 1,024 / + 64 / 1,024 it used before. + +- **First-in-first-out order no longer resets on overwrite.** `ReplacementFIFO` + now keeps a key at its original queue position when `put` overwrites it, + updating only the payload and the byte accounting. Previously the key was + re-queued at the back, so rewriting a value made an old entry look newly + inserted and pushed its eviction arbitrarily far out — a cache that is + written in place never evicted in insertion order at all. +- **A segmented-LRU read no longer re-fronts its entry.** `ReplacementSLRU::get` + records the access flag and nothing else; list position is decided by the + maintainer and by eviction. A key that is read once no longer jumps ahead of + one that was read twice, and the read is now a single index lookup instead of + two plus a list splice. +- **Constant-time list maintenance across every policy.** `ReplacementFIFO` and + `BaseLRUList` — and through it `GhostLRUList`, `ArcList` and `ReplacementArc` + — now share the intrusive doubly-linked list and node arena introduced for + the segmented LRU, replacing the `VecDeque` rescans that ran on every get, + delete and overwrite. No policy scans a key list any more. Measured with + `examples/policy_bench.rs` (64-byte values, best of three): + + | entries | FIFO churn before | after | ARC `get` before | after | + | ------: | ----------------: | ----: | ---------------: | ----: | + | 1,024 | 5.69 us | 199 ns | 4.69 us | 340 ns | + | 4,096 | 20.2 us | 362 ns | 15.9 us | 501 ns | + | 16,384 | 153 us | 597 ns | 91.5 us | 1.02 us | + + The before column grows linearly with the entry count while the after column + stays flat, which is the point; the absolute ratios vary with machine noise. + Segmented-LRU timings are unchanged within that noise, having already moved + to intrusive lists. + +- **Segmented (sharded) `ReplacementSLRU`.** The policy now hash-partitions its + index and its hot/warm/cold lists into `num_segments` segments (256 by default, + configurable with `ReplacementSLRU::with_num_segments`), each with an + independent byte budget of `capacity / num_segments`. Inserts, lookups and + eviction only touch the segment owning the key, so eviction scans a small + segment-local list instead of one global list, and no single hot list can + consume the whole budget. Segment counts are rounded up to a power of two, and + a capacity below the segment count collapses to a single segment. +- **Constant-time list maintenance in `ReplacementSLRU`.** The hot, warm and cold + lists are now intrusive doubly-linked lists over a shared node arena instead of + key deques that were rescanned on every access, so `get`, `delete` and an + overwriting `put` unlink in O(1) rather than scanning every cached key. + Measured with `examples/policy_bench.rs` (64-byte values, best of three): + + | entries | `get` before | `get` after | `delete`+re-`put` before | `delete`+re-`put` after | + | ------: | -----------: | ----------: | -----------------------: | ----------------------: | + | 1,024 | 28.8 us | 151 ns | 17.4 us | 191 ns | + | 4,096 | 110 us | 260 ns | 52.5 us | 247 ns | + | 16,384 | 1.08 ms | 536 ns | 503 us | 563 ns | + + Under eviction pressure the previous implementation could rescan a whole list + once per eviction attempt and did not finish a 16,384-entry run in 15 minutes; + the segmented policy sustains roughly 1.2 us per insert at 65,536 entries. - **Standalone open-source library.** Removed internal-only build and validation tooling under `tools/` and every reference to retired internal system names from the source, the example, the README, and the crate metadata. The crate is now a @@ -17,6 +91,89 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **Paced collection checks.** `StorageGCController::poll` runs a collection + round when one is due, at an interval set by `set_gc_check_interval_ms` (1 + second by default). The controller has no thread of its own, so a caller + drives this from its own loop, and the interval keeps the fragmentation check + off the hot path rather than rescanning on every call. Adds `set_enable_gc`, + which toggles collection without the drain that `stop` performs, and + `GC_DEFAULT_CHECK_INTERVAL_MS`. + +- **`ConcurrentReplacementSLRU`, a segmented LRU with one lock per segment.** + `ReplacementSLRU` partitions its lists but is driven through `&mut self`, so a + caller sharing it had to wrap the whole policy in one lock and the + partitioning bought nothing — the reason segmenting exists went unrealised. + The new type gives each segment its own lock, so operations on keys that hash + to different segments do not wait on each other. Key-to-segment mapping, + segment budgets and the maintainer behaviour are identical to the + single-threaded form; it is additive, so existing callers are unaffected. + + Measured with `examples/policy_bench.rs` on 8 cores, one shared workload, + best of three: + + | threads | one global lock | per-segment locks | speedup | + | ------: | --------------: | ----------------: | ------: | + | 1 | 529 ns | 727 ns | 0.73x | + | 2 | 974 ns | 436 ns | 2.24x | + | 4 | 1,659 ns | 329 ns | 5.04x | + | 8 | 2,094 ns | 305 ns | 6.87x | + + Note the single-threaded row: with no contention to relieve, the per-segment + form is *slower*, paying for the extra indirection and 256 mutexes. Reach for + it when the policy is genuinely shared; `ReplacementSLRU` remains the better + choice for single-threaded use. +- A contention sweep in `examples/policy_bench.rs` comparing the two forms + across thread counts, and tests covering segment-layout agreement between + them, four threads sharing one policy without an outer lock, eviction + reporting from every segment, and value round-trips through the shared form. + +- **Migration pacing.** `L2CachePolicy::poll` runs whichever of the access, + tail and write passes are due, at intervals set by `set_access_interval_ms`, + `set_tail_interval_ms` and `set_write_interval_ms` (1 ms, 1 s and 1 s by + default). `flush_once` still runs all three unconditionally; `poll` paces + them the way independent timers would, so a caller driving one loop does not + write to the lower tier faster than the write interval allows. Throttling + those writes is the reason the policy exists: migration must not crowd out + reads on the device. +- `L2CachePolicy` accessors for the new configuration and counters + (`async_on_access`, `use_eviction_handler`, the three intervals, and + `access_drop_count`), plus the `L2_DEFAULT_*` constants naming every default. +- Tests pinning the adaptive replacement state machine: promotion out of the + fetch data list on a hit, both ghost-list hits shifting capacity between the + fetch and active sides, a total miss dropping the fetch tail outright when + its ghost list is empty, and delete clearing a key from whichever list holds + it. The algorithm needed no change; these make that checkable. +- Tests for the migration policy: both access-record modes and the drop on a + full buffer, the eviction handler defaulting off, interval pacing versus + `flush_once`, and the factory sizing. + +- `examples/policy_bench.rs` replaces `examples/slru_shard_bench.rs` and now + probes the segmented LRU, FIFO and ARC through the same fill / read / churn + phases, so the cost of a policy can be tracked as the working set grows. +- `ReplacementFIFO::queue_len`, which reports queued entries. It equals + `get_item_num`: the queue holds no tombstones for deleted keys. +- Tests for the changed semantics: FIFO keeping its queue position across an + overwrite and leaving no tombstone behind a delete, a segmented-LRU read + marking an entry without moving it and the maintainer then promoting it from + the tail, and node recycling in `BaseLRUList` across repeated churn. + +- **Segmented-LRU maintainer.** `ReplacementSLRU` keeps each segment's hot and + warm lists within a configurable share of the segment budget (`set_hot_lru_pct` + and `set_warm_lru_pct`, defaulting to 20% and 40%), promoting entries touched + twice into the warm list and demoting the rest into the cold list. + `run_lru_maintainer_pass` sweeps every segment, each `put` maintains the + segment it touched, and `test_config_lru_maintainer` disables both. +- Segment introspection on `ReplacementSLRU`: `num_segments`, + `segment_byte_limit`, `segment_used_size`, `list_used_size`, `list_item_num` + and `segment_for_key`, with `GetSegmentUsedSize`, `GetSegmentByteLimit`, + `GetListUsedSize`, `PickSegment` and `LRUMaintainerTask` aliases. +- `examples/policy_bench.rs`, a throughput probe that sweeps the working-set + size at a fixed segment count and the segment count under eviction pressure, + reporting both a table and a JSON summary. +- Unit tests covering segment-count resolution, key distribution and per-segment + budgets, byte and item accounting across overwrite-delete-reuse churn, and the + maintainer's promote, demote and cold-eviction paths. + - Open-source project files: `LICENSE` (Apache-2.0), a rewritten `README.md`, `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, `SECURITY.md`, this changelog, issue and pull-request templates, `CODEOWNERS`, and Dependabot configuration. diff --git a/examples/policy_bench.rs b/examples/policy_bench.rs new file mode 100755 index 0000000..71d93e2 --- /dev/null +++ b/examples/policy_bench.rs @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 MatrixArkAI + +//! Throughput probe for the cache replacement policies. +//! +//! Every policy keeps its keys in intrusive doubly-linked lists over a shared +//! node arena, so a lookup or a delete unlinks an entry in constant time rather +//! than rescanning a list of keys. The segmented LRU additionally partitions +//! its index and lists into segments, each with its own byte budget, so +//! eviction only ever walks a segment-local list. +//! +//! This example measures both properties. It sweeps the working-set size at a +//! fixed shape to show that per-operation cost does not grow with the number of +//! cached entries, and it sweeps the segment count under eviction pressure. +//! +//! Keys and access orders are generated before the timed regions so the +//! measurement reflects the policy rather than key formatting, and each case is +//! repeated with the best run reported, which suppresses scheduler noise. Costs +//! are only comparable between runs of the same policy: the byte-budgeted +//! policies return a cloned buffer from `get`, while the item-budgeted one +//! returns a bool. +//! +//! ```text +//! cargo run --release --example policy_bench +//! cargo run --release --example policy_bench -- 65536 +//! cargo run --release --example policy_bench -- 65536 scaling +//! ``` +//! +//! The first argument caps the working-set sweep; passing `scaling` as the +//! second argument skips the eviction-pressure sweep. + +use matrixcache::{ + CacheBuffer, ConcurrentReplacementSLRU, ReplacementArc, ReplacementFIFO, ReplacementSLRU, +}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +const VALUE_BYTES: usize = 64; +const KEY_PREFIX: &str = "policy-bench-key-"; +const KEY_DIGITS: usize = 12; +const REPEATS: usize = 3; + +/// Bytes one entry charges against a byte budget: key plus value. +fn entry_space() -> usize { + KEY_PREFIX.len() + KEY_DIGITS + VALUE_BYTES +} + +/// Pre-generated keys plus a visit order that touches every key exactly once +/// but not in insertion order, so lookups cannot ride on list locality. +struct Workload { + keys: Vec, + order: Vec, +} + +impl Workload { + fn new(entries: usize) -> Self { + let keys = (0..entries) + .map(|index| format!("{KEY_PREFIX}{index:012}")) + .collect(); + // Multiplying by an odd constant modulo a power of two is a bijection, + // so every index is visited exactly once when `entries` is a power of + // two, and the order is still well spread otherwise. + let order = (0..entries) + .map(|index| index.wrapping_mul(2_654_435_761) % entries.max(1)) + .collect(); + Self { keys, order } + } +} + +fn buffer_for(key: &str) -> CacheBuffer { + let mut buffer = CacheBuffer::new(vec![b'v'; VALUE_BYTES]); + buffer.SetKey(key); + buffer +} + +fn ns_per_op(elapsed: Duration, ops: usize) -> f64 { + if ops == 0 { + return 0.0; + } + elapsed.as_nanos() as f64 / ops as f64 +} + +#[derive(Debug, Clone, Copy)] +struct CaseResult { + put_ns: f64, + get_ns: f64, + churn_ns: f64, + evicted: usize, + resident: usize, +} + +impl CaseResult { + fn best(self, other: CaseResult) -> CaseResult { + CaseResult { + put_ns: self.put_ns.min(other.put_ns), + get_ns: self.get_ns.min(other.get_ns), + churn_ns: self.churn_ns.min(other.churn_ns), + evicted: other.evicted, + resident: other.resident, + } + } +} + +/// Byte capacity for `entries` items, divided by `pressure`: a value above four +/// forces eviction on nearly every insert. +fn byte_capacity(entries: usize, pressure: usize) -> usize { + (entries * entry_space() * 4 / pressure.max(1)).max(entry_space() * 4) +} + +fn run_slru(workload: &Workload, segments: usize, pressure: usize) -> CaseResult { + let entries = workload.keys.len(); + let mut policy = ReplacementSLRU::with_num_segments(byte_capacity(entries, pressure), segments); + policy.init().expect("init segmented lru"); + + let mut evicted = 0usize; + let started = Instant::now(); + for key in &workload.keys { + evicted += policy.put(buffer_for(key)).len(); + } + let put_elapsed = started.elapsed(); + + let started = Instant::now(); + for &index in &workload.order { + let _ = policy.get(&workload.keys[index]); + } + let get_elapsed = started.elapsed(); + + let churn = entries / 2; + let started = Instant::now(); + for &index in workload.order.iter().take(churn) { + let key = &workload.keys[index]; + let _ = policy.delete(key); + evicted += policy.put(buffer_for(key)).len(); + } + let churn_elapsed = started.elapsed(); + + CaseResult { + put_ns: ns_per_op(put_elapsed, entries), + get_ns: ns_per_op(get_elapsed, entries), + churn_ns: ns_per_op(churn_elapsed, churn * 2), + evicted, + resident: policy.get_item_num(), + } +} + +fn run_fifo(workload: &Workload, pressure: usize) -> CaseResult { + let entries = workload.keys.len(); + let mut policy = ReplacementFIFO::new(byte_capacity(entries, pressure)); + policy.init().expect("init fifo"); + + let mut evicted = 0usize; + let started = Instant::now(); + for key in &workload.keys { + evicted += policy.put(buffer_for(key)).len(); + } + let put_elapsed = started.elapsed(); + + let started = Instant::now(); + for &index in &workload.order { + let _ = policy.get(&workload.keys[index]); + } + let get_elapsed = started.elapsed(); + + // Overwriting in place is the interesting churn for this policy: it must + // keep the queue position while replacing the payload. + let churn = entries / 2; + let started = Instant::now(); + for &index in workload.order.iter().take(churn) { + let key = &workload.keys[index]; + let _ = policy.delete(key); + evicted += policy.put(buffer_for(key)).len(); + } + let churn_elapsed = started.elapsed(); + + CaseResult { + put_ns: ns_per_op(put_elapsed, entries), + get_ns: ns_per_op(get_elapsed, entries), + churn_ns: ns_per_op(churn_elapsed, churn * 2), + evicted, + resident: policy.get_item_num(), + } +} + +fn run_arc(workload: &Workload, pressure: usize) -> CaseResult { + let entries = workload.keys.len(); + // This policy budgets by item count, not bytes. + let mut policy = ReplacementArc::new((entries * 4 / pressure.max(1)).max(4)); + policy.init().expect("init arc"); + + let started = Instant::now(); + for key in &workload.keys { + policy.put(key.clone()); + } + let put_elapsed = started.elapsed(); + + let started = Instant::now(); + for &index in &workload.order { + let _ = policy.get(&workload.keys[index]); + } + let get_elapsed = started.elapsed(); + + let churn = entries / 2; + let started = Instant::now(); + for &index in workload.order.iter().take(churn) { + let key = &workload.keys[index]; + let _ = policy.delete(key); + policy.put(key.clone()); + } + let churn_elapsed = started.elapsed(); + + CaseResult { + put_ns: ns_per_op(put_elapsed, entries), + get_ns: ns_per_op(get_elapsed, entries), + churn_ns: ns_per_op(churn_elapsed, churn * 2), + evicted: 0, + resident: policy.get_active_tail(entries).len(), + } +} + +/// Operations each thread performs in the contention sweep. +const CONTENTION_OPS_PER_THREAD: usize = 8_192; + +/// Drive the segmented policy from `threads` threads behind one lock over the +/// whole policy. This is what a caller has to do to share the `&mut self` form, +/// and it serialises every operation no matter how the keys are partitioned. +fn run_contention_global(keys: &[String], threads: usize) -> f64 { + let capacity = keys.len() * entry_space() * 4; + let policy = Mutex::new({ + let mut inner = ReplacementSLRU::with_num_segments(capacity, 256); + inner.init().expect("init segmented lru"); + inner + }); + let per_thread = keys.len() / threads; + + let started = Instant::now(); + std::thread::scope(|scope| { + for thread in 0..threads { + let policy = &policy; + let slice = &keys[thread * per_thread..(thread + 1) * per_thread]; + scope.spawn(move || { + for key in slice { + policy.lock().expect("policy lock").put(buffer_for(key)); + let _ = policy.lock().expect("policy lock").get(key); + } + }); + } + }); + ns_per_op(started.elapsed(), per_thread * threads * 2) +} + +/// The same workload against per-segment locks. Keys that hash to different +/// segments proceed in parallel. +fn run_contention_sharded(keys: &[String], threads: usize, segments: usize) -> f64 { + let capacity = keys.len() * entry_space() * 4; + let policy = ConcurrentReplacementSLRU::with_num_segments(capacity, segments); + policy.init().expect("init segmented lru"); + let per_thread = keys.len() / threads; + + let started = Instant::now(); + std::thread::scope(|scope| { + for thread in 0..threads { + let policy = &policy; + let slice = &keys[thread * per_thread..(thread + 1) * per_thread]; + scope.spawn(move || { + for key in slice { + policy.put(buffer_for(key)); + let _ = policy.get(key); + } + }); + } + }); + ns_per_op(started.elapsed(), per_thread * threads * 2) +} + +fn best_ns f64>(run: F) -> f64 { + let mut best = run(); + for _ in 1..REPEATS { + best = best.min(run()); + } + best +} + +fn best_of CaseResult>(run: F) -> CaseResult { + let mut best = run(); + for _ in 1..REPEATS { + best = best.best(run()); + } + best +} + +fn print_row(label: &str, entries: usize, shape: &str, result: CaseResult) { + println!( + "{label:<10} {entries:>8} {shape:>9} {:>12.1} {:>12.1} {:>13.1} {:>10} {:>10}", + result.put_ns, result.get_ns, result.churn_ns, result.evicted, result.resident, + ); +} + +fn json_case(label: &str, entries: usize, shape: &str, result: CaseResult) -> String { + format!( + "{{\"case\":\"{label}\",\"entries\":{entries},\"shape\":\"{shape}\",\ +\"put_ns_per_op\":{:.1},\"get_ns_per_op\":{:.1},\"churn_ns_per_op\":{:.1},\ +\"evicted\":{},\"resident\":{}}}", + result.put_ns, result.get_ns, result.churn_ns, result.evicted, result.resident, + ) +} + +fn main() { + let max_entries: usize = std::env::args() + .nth(1) + .and_then(|value| value.parse().ok()) + .unwrap_or(65_536); + let scaling_only = std::env::args() + .nth(2) + .is_some_and(|mode| mode == "scaling"); + + let mut sizes = Vec::new(); + let mut size = 1_024usize; + while size <= max_entries { + sizes.push(size); + size *= 4; + } + if sizes.is_empty() { + sizes.push(max_entries.max(1)); + } + + // Warm the allocator and the CPU before the first measured case. + let _ = run_slru(&Workload::new(1_024), 256, 1); + + println!( + "{:<10} {:>8} {:>9} {:>12} {:>12} {:>13} {:>10} {:>10}", + "policy", + "entries", + "shape", + "put ns/op", + "get ns/op", + "churn ns/op", + "evicted", + "resident" + ); + + let mut cases = Vec::new(); + + // Scaling sweep: per-operation cost against working-set size, with enough + // headroom that inserts do not evict. Intrusive links keep every phase + // flat; rescanning a key list would make get and churn grow with the count. + for &entries in &sizes { + let workload = Workload::new(entries); + + let result = best_of(|| run_slru(&workload, 256, 1)); + print_row("slru", entries, "256 seg", result); + cases.push(json_case("slru", entries, "256 seg", result)); + + let result = best_of(|| run_fifo(&workload, 1)); + print_row("fifo", entries, "-", result); + cases.push(json_case("fifo", entries, "-", result)); + + let result = best_of(|| run_arc(&workload, 1)); + print_row("arc", entries, "-", result); + cases.push(json_case("arc", entries, "-", result)); + } + + // Segment sweep at a fixed working set with the capacity tightened so that + // inserts evict: more segments means a smaller list per eviction. + if !scaling_only { + let pressure_entries = sizes.last().copied().unwrap_or(1_024); + let workload = Workload::new(pressure_entries); + for &segments in &[1usize, 8, 64, 256, 1_024] { + let result = best_of(|| run_slru(&workload, segments, 8)); + let shape = format!("{segments} seg"); + print_row("slru/press", pressure_entries, &shape, result); + cases.push(json_case("slru-pressure", pressure_entries, &shape, result)); + } + } + + // Contention sweep: the same shared workload behind one lock over the whole + // policy versus one lock per segment. Sharding only pays off once the locks + // are per segment, so this is what the partitioning is actually for. + if !scaling_only { + println!(); + println!( + "{:<10} {:>8} {:>9} {:>12} {:>12} {:>13}", + "contention", "threads", "shape", "global ns/op", "shard ns/op", "speedup" + ); + for &threads in &[1usize, 2, 4, 8] { + let keys = Workload::new(threads * CONTENTION_OPS_PER_THREAD).keys; + let global = best_ns(|| run_contention_global(&keys, threads)); + let sharded = best_ns(|| run_contention_sharded(&keys, threads, 256)); + let speedup = if sharded > 0.0 { global / sharded } else { 0.0 }; + println!( + "{:<10} {threads:>8} {:>9} {global:>12.1} {sharded:>12.1} {speedup:>12.2}x", + "lock", "256 seg" + ); + cases.push(format!( + "{{\"case\":\"contention\",\"threads\":{threads},\"num_segments\":256,\ +\"global_lock_ns_per_op\":{global:.1},\"per_segment_lock_ns_per_op\":{sharded:.1},\ +\"speedup\":{speedup:.2}}}" + )); + } + } + + println!(); + println!("{{\"benchmark\":\"policy_bench\",\"value_bytes\":{VALUE_BYTES},\"cases\":["); + for (index, case) in cases.iter().enumerate() { + let comma = if index + 1 == cases.len() { "" } else { "," }; + println!(" {case}{comma}"); + } + println!("]}}"); +} diff --git a/src/core/storage_config.rs b/src/core/storage_config.rs index 849063e..64e469d 100644 --- a/src/core/storage_config.rs +++ b/src/core/storage_config.rs @@ -1436,6 +1436,37 @@ impl CacheEvictionCallback { } } +/// Receives the number of entries evicted from one tier in a single batch. +/// +/// Eviction metrics are independent of the eviction handler: they are reported +/// even while the handler is disabled. Counting entries rather than bytes is +/// what makes that affordable, since a count needs nothing materialised. +#[derive(Clone)] +struct CacheEvictionMetricCallback { + callback: Arc, +} + +impl std::fmt::Debug for CacheEvictionMetricCallback { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("CacheEvictionMetricCallback") + } +} + +impl CacheEvictionMetricCallback { + fn new(callback: F) -> Self + where + F: Fn(CacheTier, usize) + Send + Sync + 'static, + { + Self { + callback: Arc::new(callback), + } + } + + fn call(&self, tier: CacheTier, count: usize) { + (self.callback)(tier, count); + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CacheBlockKind { Page, diff --git a/src/runtime/allocators_executors.rs b/src/runtime/allocators_executors.rs index eed1dc4..3f176e1 100644 --- a/src/runtime/allocators_executors.rs +++ b/src/runtime/allocators_executors.rs @@ -1632,6 +1632,9 @@ impl CacheExecutor { } } +/// Milliseconds between collection checks in [`StorageGCController::poll`]. +pub const GC_DEFAULT_CHECK_INTERVAL_MS: u64 = 1_000; + #[derive(Debug, Clone)] pub struct StorageGCController { allocator: SimpleLogBasedMemoryAllocator, @@ -1640,6 +1643,8 @@ pub struct StorageGCController { enable_gc: bool, free_mem_min: usize, fragmentation_ratio_max: u8, + gc_check_interval_ms: u64, + last_gc_check: Option, complete_gc_chunks: i64, fly_gc_chunks: i64, complete_gc_tasks: i64, @@ -1663,6 +1668,8 @@ impl StorageGCController { enable_gc: false, free_mem_min, fragmentation_ratio_max, + gc_check_interval_ms: GC_DEFAULT_CHECK_INTERVAL_MS, + last_gc_check: None, complete_gc_chunks: 0, fly_gc_chunks: 0, complete_gc_tasks: 0, @@ -1682,6 +1689,46 @@ impl StorageGCController { self.pause_gc = pause; } + pub fn gc_check_interval_ms(&self) -> u64 { + self.gc_check_interval_ms + } + + pub fn set_gc_check_interval_ms(&mut self, interval_ms: u64) { + self.gc_check_interval_ms = interval_ms; + } + + /// Enable or disable collection without draining outstanding work, unlike + /// [`StorageGCController::stop`]. + pub fn set_enable_gc(&mut self, enable: bool) { + self.enable_gc = enable; + } + + /// Check whether collection is due and, if so, run one round. + /// + /// This is the monitoring half of the controller: it decides *when* to + /// look, while `pick_submit_chunks` decides what to reclaim once asked. + /// Because the controller has no thread of its own, a caller drives this + /// from its own loop and the interval keeps the fragmentation check off + /// the hot path rather than running it on every call. Returns the number + /// of chunks reclaimed. + pub fn poll(&mut self) -> Result { + if !self.enable_gc || self.pause_gc { + return Ok(0); + } + let now = Instant::now(); + let due = match self.last_gc_check { + None => true, + Some(last) => { + now.duration_since(last) >= Duration::from_millis(self.gc_check_interval_ms) + } + }; + if !due { + return Ok(0); + } + self.last_gc_check = Some(now); + self.pick_submit_chunks() + } + // Completion barrier: `fly_gc_chunks` is maintained by the GC executor's // submit/complete accounting, not mutated in this spin body. #[allow(clippy::while_immutable_condition)] diff --git a/src/runtime/builder_and_gc.rs b/src/runtime/builder_and_gc.rs index 7b854ae..f6723d9 100644 --- a/src/runtime/builder_and_gc.rs +++ b/src/runtime/builder_and_gc.rs @@ -689,7 +689,6 @@ impl CacheInner { } } self.pmem.clear(); - self.pmem_order.clear(); self.pmem_fifo_order.clear(); self.pmem_bytes = 0; for (key, expected_len) in live { @@ -762,8 +761,7 @@ impl CacheInner { }); } self.disk_index = recovered_index; - self.disk_order = recovered_order.clone(); - self.disk_fifo_order = recovered_order; + self.disk_fifo_order = recovered_order.into_iter().collect(); self.ssd_bytes = recovered_bytes; self.stats.disk_bytes = recovered_bytes; Ok(report) @@ -773,7 +771,6 @@ impl CacheInner { let manifest_path = self.manifest_path(); if !manifest_path.exists() { self.disk_index.clear(); - self.disk_order.clear(); self.ssd_bytes = 0; self.stats.disk_bytes = 0; return Ok(CacheRecoverReport::default()); @@ -839,8 +836,7 @@ impl CacheInner { } self.disk_index = recovered_index; - self.disk_order = recovered_order.clone(); - self.disk_fifo_order = recovered_order; + self.disk_fifo_order = recovered_order.into_iter().collect(); self.ssd_bytes = recovered_bytes; self.stats.disk_bytes = recovered_bytes; Ok(report) @@ -930,51 +926,6 @@ impl CacheInner { fn record_hit(&mut self, key: &CacheKey, block_bytes: usize) { self.record_hit_metadata(key, block_bytes); - self.touch_hit_queues_for_key(key); - } - - fn touch_hit_queues_for_key(&mut self, key: &CacheKey) { - if self.disk_index.contains_key(key) && self.disk_order.back() != Some(key) { - self.disk_order.retain(|candidate| candidate != key); - self.disk_order.push_back(key.clone()); - } - if self.memory.contains_key(key) && self.order.back() != Some(key) { - self.order.retain(|candidate| candidate != key); - self.order.push_back(key.clone()); - } - } - - fn touch_queue_batch(queue: &mut VecDeque, keys: &[CacheKey]) { - if keys.is_empty() { - return; - } - let mut seen = HashSet::new(); - let mut ordered = Vec::new(); - for key in keys.iter().rev() { - if seen.insert(key.clone()) { - ordered.push(key.clone()); - } - } - ordered.reverse(); - const SET_MEMBERSHIP_THRESHOLD: usize = 8; - if ordered.len() > SET_MEMBERSHIP_THRESHOLD { - let key_set = ordered.iter().cloned().collect::>(); - queue.retain(|candidate| !key_set.contains(candidate)); - } else { - queue.retain(|candidate| !ordered.contains(candidate)); - } - queue.extend(ordered); - } - - fn touch_hit_queues_batch( - &mut self, - disk_keys: &[CacheKey], - memory_keys: &[CacheKey], - pmem_keys: &[CacheKey], - ) { - Self::touch_queue_batch(&mut self.disk_order, disk_keys); - Self::touch_queue_batch(&mut self.order, memory_keys); - Self::touch_queue_batch(&mut self.pmem_order, pmem_keys); } fn put_memory(&mut self, key: CacheKey, value: Vec) -> bool { @@ -989,10 +940,8 @@ impl CacheInner { let value = Arc::<[u8]>::from(value); if let Some(old) = self.memory.insert(key.clone(), Arc::clone(&value)) { self.memory_bytes = self.memory_bytes.saturating_sub(old.len()); - self.touch_key(&key); } else { - self.order.push_back(key.clone()); - self.memory_fifo_order.push_back(key); + self.memory_fifo_order.push_back_if_absent(key); } self.memory_bytes += value.len(); self.evict_memory_to_capacity_since(eviction_started); @@ -1027,10 +976,8 @@ impl CacheInner { let value = Arc::<[u8]>::from(value); if let Some(old) = self.pmem.insert(key.clone(), Arc::clone(&value)) { self.pmem_bytes = self.pmem_bytes.saturating_sub(old.len()); - self.touch_key(&key); } else { - self.pmem_order.push_back(key.clone()); - self.pmem_fifo_order.push_back(key); + self.pmem_fifo_order.push_back_if_absent(key); } self.pmem_bytes = self.pmem_bytes.saturating_add(value.len()); self.evict_pmem_to_capacity_since(eviction_started); @@ -1074,8 +1021,7 @@ impl CacheInner { return false; } self.disk_index.insert(key.clone(), block_len as u64); - self.disk_order.push_back(key.clone()); - self.disk_fifo_order.push_back(key.clone()); + self.disk_fifo_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len as u64); self.stats.disk_fills = self.stats.disk_fills.saturating_add(1); self.stats.ssd_admission_accepted = self.stats.ssd_admission_accepted.saturating_add(1); @@ -1124,17 +1070,6 @@ impl CacheInner { self.put_memory(key, value) } - fn touch_key(&mut self, key: &CacheKey) { - if self.memory.contains_key(key) && self.order.back() != Some(key) { - self.order.retain(|candidate| candidate != key); - self.order.push_back(key.clone()); - } - if self.pmem.contains_key(key) && self.pmem_order.back() != Some(key) { - self.pmem_order.retain(|candidate| candidate != key); - self.pmem_order.push_back(key.clone()); - } - } - fn evict_memory_to_capacity(&mut self) { self.evict_memory_to_capacity_since(Instant::now()); } @@ -1184,8 +1119,6 @@ impl CacheInner { } if !victim_keys.is_empty() { let victim_key_set = victim_keys.iter().cloned().collect::>(); - self.order - .retain(|candidate| !victim_key_set.contains(candidate)); self.memory_fifo_order .retain(|candidate| !victim_key_set.contains(candidate)); } @@ -1233,8 +1166,6 @@ impl CacheInner { } if !victim_keys.is_empty() { let victim_key_set = victim_keys.iter().cloned().collect::>(); - self.pmem_order - .retain(|candidate| !victim_key_set.contains(candidate)); self.pmem_fifo_order .retain(|candidate| !victim_key_set.contains(candidate)); } @@ -1253,6 +1184,7 @@ impl CacheInner { fn evict_ssd_for_batch(&mut self, incoming_bytes: u64) { let mut victim_keys = Vec::new(); while self.ssd_bytes.saturating_add(incoming_bytes) > self.ssd_capacity_bytes as u64 { + let before = self.ssd_bytes; let Some((victim, reason, pinned_skips)) = self.select_ssd_eviction_victim() else { break; }; @@ -1262,6 +1194,10 @@ impl CacheInner { .saturating_add(pinned_skips); let evicted_value = self.read_ssd_value_for_eviction(&victim); let removed_bytes = self.disk_index.remove(&victim).unwrap_or_default(); + // Drop the victim from the order as it is taken. Selection reads + // that order, so leaving it until after the loop lets the next + // round pick the same key, whose bytes are already gone. + self.disk_fifo_order.remove(&victim); self.record_eviction(CacheTier::Ssd, victim.clone(), evicted_value.clone()); self.ssd_bytes = self.ssd_bytes.saturating_sub(removed_bytes); self.stats.ssd_evictions = self.stats.ssd_evictions.saturating_add(1); @@ -1272,14 +1208,17 @@ impl CacheInner { self.metadata.remove(&victim); } victim_keys.push(victim); + if self.ssd_bytes == before { + // Freed nothing this round; the memory and pmem loops bail out + // here too rather than spin. + break; + } } if victim_keys.is_empty() { self.stats.disk_bytes = self.ssd_bytes; return; } let victim_key_set = victim_keys.iter().cloned().collect::>(); - self.disk_order - .retain(|candidate| !victim_key_set.contains(candidate)); self.disk_fifo_order .retain(|candidate| !victim_key_set.contains(candidate)); let _ = self.delete_ssd_blocks(&victim_keys); @@ -1327,10 +1266,10 @@ impl CacheInner { fn select_fifo_eviction_victim( &self, - keys: &VecDeque, + keys: &CacheKeyOrder, ) -> Option<(CacheKey, EvictionReason, u64)> { let mut pinned_skips = 0u64; - for key in keys { + for key in keys.iter() { if self.pinned.contains_key(key) { pinned_skips = pinned_skips.saturating_add(1); continue; @@ -1458,6 +1397,11 @@ impl CacheInner { } fn record_eviction(&mut self, tier: CacheTier, key: CacheKey, value: Vec) { + // The metric counts evictions and needs no value, so it is recorded + // even while the eviction handler is disabled. + if self.eviction_metric_callback.is_some() { + self.pending_eviction_metric_tiers.push_back(tier); + } if self.eviction_handler_enabled && self.eviction_callback.is_some() { self.pending_eviction_records .push_back(CacheEvictionRecord { tier, key, value }); @@ -1508,8 +1452,7 @@ impl CacheInner { removed_pinned_bytes = removed_pinned_bytes.max(value.len()); self.memory_bytes = self.memory_bytes.saturating_sub(value.len()); } - self.order.retain(|candidate| candidate != key); - self.memory_fifo_order.retain(|candidate| candidate != key); + self.memory_fifo_order.remove(key); if remove_disk { let disk_bytes = self.disk_index.remove(key).unwrap_or_default(); removed_pinned_bytes = removed_pinned_bytes.max( @@ -1536,8 +1479,7 @@ impl CacheInner { if remove_disk { let _ = self.persist_pmem_delete(key); } - self.pmem_order.retain(|candidate| candidate != key); - self.pmem_fifo_order.retain(|candidate| candidate != key); + self.pmem_fifo_order.remove(key); self.metadata.remove(key); if key_pinned && removed_pinned_bytes > 0 { self.pinned_removed_bytes @@ -1551,14 +1493,10 @@ impl CacheInner { if remove_disk { match key_set.as_ref() { Some(key_set) => { - self.disk_order - .retain(|candidate| !key_set.contains(candidate)); self.disk_fifo_order .retain(|candidate| !key_set.contains(candidate)); } None => { - self.disk_order - .retain(|candidate| !keys.contains(candidate)); self.disk_fifo_order .retain(|candidate| !keys.contains(candidate)); } @@ -1577,11 +1515,8 @@ impl CacheInner { self.pmem.clear(); self.clear_pmem_persistence()?; self.disk_index.clear(); - self.disk_order.clear(); self.disk_fifo_order.clear(); - self.order.clear(); self.memory_fifo_order.clear(); - self.pmem_order.clear(); self.pmem_fifo_order.clear(); self.pinned.clear(); self.pinned_handle_bytes.clear(); @@ -1591,6 +1526,7 @@ impl CacheInner { self.async_writeback_positions.clear(); self.async_writeback_queue_bytes = 0; self.pending_eviction_records.clear(); + self.pending_eviction_metric_tiers.clear(); self.memory_bytes = 0; self.pmem_bytes = 0; self.ssd_bytes = 0; diff --git a/src/runtime/cache_facades.rs b/src/runtime/cache_facades.rs index d3314cc..2746653 100644 --- a/src/runtime/cache_facades.rs +++ b/src/runtime/cache_facades.rs @@ -266,13 +266,6 @@ impl CacheInstance { if record.tier != tier { return; } - if let Some(callback) = eviction_metric_callback - .read() - .expect("cache instance metric callback lock poisoned") - .clone() - { - callback(record.value.len()); - } if let Some(callback) = eviction_callback .read() .expect("cache instance eviction callback lock poisoned") @@ -281,6 +274,19 @@ impl CacheInstance { callback(record); } }); + self.cache + .register_eviction_metric_callback(move |record_tier, count| { + if record_tier != tier { + return; + } + if let Some(callback) = eviction_metric_callback + .read() + .expect("cache instance metric callback lock poisoned") + .clone() + { + callback(count); + } + }); } pub fn start(&self) -> Result<(), CacheError> { @@ -926,11 +932,39 @@ impl L1CacheInterface for L1CacheImplement { } } +/// Milliseconds between access-record drain passes. +pub const L2_DEFAULT_ACCESS_INTERVAL_MS: u64 = 1; +/// Milliseconds between passes that pull tail keys out of the upper tier. +pub const L2_DEFAULT_TAIL_INTERVAL_MS: u64 = 1_000; +/// Milliseconds between passes that drain the lower-tier write queue. +pub const L2_DEFAULT_WRITE_INTERVAL_MS: u64 = 1_000; +/// Access records buffered before further records are dropped. +pub const L2_DEFAULT_ACCESS_BUFFER_CAPACITY: usize = 100_000; +/// Keys pulled from a tail in one pass. +pub const L2_DEFAULT_TAIL_BATCH_SIZE: usize = 1_000; +/// Buffers queued for the lower tier before enqueue starts failing. +pub const L2_DEFAULT_WRITE_BUFFER_CAPACITY: usize = 10_000; +/// Item capacity of the adaptive policy that decides migration order. +pub const L2_DEFAULT_MAX_ARC_CACHE_ITEMS: usize = 100_000; +/// Whether access records are buffered rather than applied inline. +pub const L2_DEFAULT_ASYNC_ON_ACCESS: bool = true; +/// Whether an evicted buffer is queued for the lower tier or dropped. +pub const L2_DEFAULT_USE_EVICTION_HANDLER: bool = false; + pub struct L2CachePolicy { l1_cache: L1CacheImplement, l2_cache: CacheInstance, arc_policy: ReplacementArc, tail_batch_size: usize, + access_interval_ms: u64, + tail_interval_ms: u64, + write_interval_ms: u64, + async_on_access: bool, + use_eviction_handler: bool, + last_access_pass: Option, + last_tail_pass: Option, + last_write_pass: Option, + access_drop_count: u64, access_queue: VecDeque<(AccessRecordType, String)>, write_buffer_queue: VecDeque, access_buffer_capacity: usize, @@ -962,6 +996,15 @@ impl L2CachePolicy { l2_cache, arc_policy, tail_batch_size, + access_interval_ms: L2_DEFAULT_ACCESS_INTERVAL_MS, + tail_interval_ms: L2_DEFAULT_TAIL_INTERVAL_MS, + write_interval_ms: L2_DEFAULT_WRITE_INTERVAL_MS, + async_on_access: L2_DEFAULT_ASYNC_ON_ACCESS, + use_eviction_handler: L2_DEFAULT_USE_EVICTION_HANDLER, + last_access_pass: None, + last_tail_pass: None, + last_write_pass: None, + access_drop_count: 0, access_queue: VecDeque::with_capacity(access_buffer_capacity), write_buffer_queue: VecDeque::with_capacity(write_buffer_capacity), access_buffer_capacity, @@ -988,19 +1031,113 @@ impl L2CachePolicy { self.stopped = true; } + pub fn access_interval_ms(&self) -> u64 { + self.access_interval_ms + } + + pub fn set_access_interval_ms(&mut self, interval_ms: u64) { + self.access_interval_ms = interval_ms; + } + + pub fn tail_interval_ms(&self) -> u64 { + self.tail_interval_ms + } + + pub fn set_tail_interval_ms(&mut self, interval_ms: u64) { + self.tail_interval_ms = interval_ms; + } + + pub fn write_interval_ms(&self) -> u64 { + self.write_interval_ms + } + + pub fn set_write_interval_ms(&mut self, interval_ms: u64) { + self.write_interval_ms = interval_ms; + } + + pub fn async_on_access(&self) -> bool { + self.async_on_access + } + + /// When set, `on_access` buffers the record instead of applying it to the + /// migration-order policy inline. Buffering keeps the caller off that + /// update path, at the cost of the policy lagging behind the workload; + /// records arriving once the buffer is full are dropped and counted by + /// [`L2CachePolicy::access_drop_count`]. + pub fn set_async_on_access(&mut self, async_on_access: bool) { + self.async_on_access = async_on_access; + } + + pub fn use_eviction_handler(&self) -> bool { + self.use_eviction_handler + } + + /// When set, a buffer handed to `on_evict` is queued for the lower tier. + /// Off by default, so an eviction drops the data instead of writing it — + /// the tail passes are then the only path into the lower tier. + pub fn set_use_eviction_handler(&mut self, use_eviction_handler: bool) { + self.use_eviction_handler = use_eviction_handler; + } + + /// Access records dropped because the buffer was full. + pub fn access_drop_count(&self) -> u64 { + self.access_drop_count + } + + /// Run whichever passes are due, honouring the configured intervals. + /// + /// This is the scheduling half of the policy. `flush_once` runs all three + /// passes unconditionally; `poll` paces them the way independent timers + /// would, so a caller driving it from one loop does not write to the lower + /// tier faster than the write interval allows — the throttling exists to + /// keep migration writes from crowding out reads on the device. Returns + /// the number of buffers written by this call. + pub fn poll(&mut self) -> Result { + if self.stopped || self.paused { + return Ok(0); + } + let now = Instant::now(); + if Self::pass_due(self.last_access_pass, now, self.access_interval_ms) { + self.last_access_pass = Some(now); + self.access_task_internal(); + } + if Self::pass_due(self.last_tail_pass, now, self.tail_interval_ms) { + self.last_tail_pass = Some(now); + self.tail_task_internal(); + } + if Self::pass_due(self.last_write_pass, now, self.write_interval_ms) { + self.last_write_pass = Some(now); + return self.write_task_internal(); + } + Ok(0) + } + + fn pass_due(last: Option, now: Instant, interval_ms: u64) -> bool { + match last { + None => true, + Some(last) => now.duration_since(last) >= Duration::from_millis(interval_ms), + } + } + pub fn on_access(&mut self, record_type: AccessRecordType, key: &str) { if self.stopped { return; } - if self.access_buffer_capacity == 0 || self.access_queue.len() < self.access_buffer_capacity + if !self.async_on_access { + self.do_access(record_type, key.to_string()); + return; + } + if self.access_buffer_capacity != 0 + && self.access_queue.len() >= self.access_buffer_capacity { - self.access_queue.push_back((record_type, key.to_string())); + self.access_drop_count = self.access_drop_count.saturating_add(1); + return; } - self.access_task_internal(); + self.access_queue.push_back((record_type, key.to_string())); } pub fn on_evict(&mut self, cache_buffer: CacheBuffer) { - if self.stopped { + if self.stopped || !self.use_eviction_handler { return; } self.put_queue(cache_buffer); @@ -1094,14 +1231,22 @@ impl L2CachePolicy { } } + /// Drain the write queue. + /// + /// A buffer that fails to write is counted and skipped rather than + /// aborting the drain, so one bad key cannot stall every entry queued + /// behind it. Failures stay visible through + /// [`L2CachePolicy::write_fail_count`]. Returns how many buffers were + /// handled, which includes keys already present in the lower tier. pub fn write_task_internal(&mut self) -> Result { if self.stopped || self.paused { return Ok(0); } let mut written = 0usize; while let Some(buffer) = self.fetch_queue() { - self.do_one_write(buffer)?; - written = written.saturating_add(1); + if self.do_one_write(buffer).is_ok() { + written = written.saturating_add(1); + } } Ok(written) } @@ -1229,9 +1374,16 @@ impl L2CachePolicyFactory { l1_cache: L1CacheImplement, l2_cache: CacheInstance, ) -> L2CachePolicy { - let mut arc_policy = ReplacementArc::new(1024); + let mut arc_policy = ReplacementArc::new(L2_DEFAULT_MAX_ARC_CACHE_ITEMS); let _ = arc_policy.Init(); - L2CachePolicy::new(l1_cache, l2_cache, arc_policy, 1024, 64, 1024) + L2CachePolicy::new( + l1_cache, + l2_cache, + arc_policy, + L2_DEFAULT_ACCESS_BUFFER_CAPACITY, + L2_DEFAULT_TAIL_BATCH_SIZE, + L2_DEFAULT_WRITE_BUFFER_CAPACITY, + ) } #[allow(non_snake_case)] @@ -1739,7 +1891,7 @@ struct SimpleLruEntry { struct SimpleLruInner { capacity: usize, size: usize, - order: VecDeque, + order: CacheKeyOrder, entries: HashMap, } @@ -1752,7 +1904,7 @@ impl SimpleLruInner { Self { capacity, size: 0, - order: VecDeque::new(), + order: CacheKeyOrder::new(), entries: HashMap::new(), } } @@ -1779,7 +1931,7 @@ impl SimpleLruInner { fn remove(&mut self, key: &CacheKey) { if let Some(entry) = self.entries.remove(key) { self.size = self.size.saturating_sub(entry.size); - self.order.retain(|candidate| candidate != key); + self.order.remove(key); } } @@ -1799,23 +1951,32 @@ impl SimpleLruInner { } fn touch(&mut self, key: &CacheKey) { - self.order.retain(|candidate| candidate != key); + // Moves the key to the front if it is already tracked, so a lookup + // costs the same whether the cache holds ten entries or ten million. self.order.push_front(key.clone()); } fn evict_unpinned(&mut self) { while self.size > self.capacity { - let Some(index) = self.order.iter().rposition(|key| { - self.entries - .get(key) - .is_some_and(|entry| Arc::strong_count(&entry.value) == 1) - }) else { + // Walk from the least recently used end and stop at the first + // entry nobody is holding. Scanning forward for the last match + // visited every entry on every eviction; from this end the usual + // case stops immediately. + let victim = self + .order + .iter_rev() + .find(|key| { + self.entries + .get(key) + .is_some_and(|entry| Arc::strong_count(&entry.value) == 1) + }) + .cloned(); + let Some(key) = victim else { break; }; - if let Some(key) = self.order.remove(index) { - if let Some(entry) = self.entries.remove(&key) { - self.size = self.size.saturating_sub(entry.size); - } + self.order.remove(&key); + if let Some(entry) = self.entries.remove(&key) { + self.size = self.size.saturating_sub(entry.size); } } } @@ -2808,3 +2969,303 @@ impl StringCacheApi for MultiTierCache { self.cache.Size() } } + +const CACHE_ORDER_NIL: u32 = u32::MAX; + +#[derive(Debug, Clone)] +struct CacheOrderNode { + key: CacheKey, + prev: u32, + next: u32, +} + +/// Recency ordering over cache keys, from least recently used at the front to +/// most recently used at the back. +/// +/// A `VecDeque` has to be rescanned to move a key to the back, so a +/// cache hit costs O(n) in the number of resident entries. This keeps the same +/// ordering in an intrusive doubly-linked list over a node arena, with an index +/// from key to node, so recording a hit is O(1) regardless of how much is +/// cached. Freed nodes are recycled, so churn does not grow the arena. +/// +/// Bulk removal by predicate is still a scan; those run on invalidation paths, +/// not on a hit. +#[derive(Debug, Clone)] +pub struct CacheKeyOrder { + nodes: Vec, + free: Vec, + index: HashMap, + head: u32, + tail: u32, +} + +impl Default for CacheKeyOrder { + fn default() -> Self { + Self::new() + } +} + +impl CacheKeyOrder { + pub fn new() -> Self { + Self { + nodes: Vec::new(), + free: Vec::new(), + index: HashMap::new(), + head: CACHE_ORDER_NIL, + tail: CACHE_ORDER_NIL, + } + } + + pub fn len(&self) -> usize { + self.index.len() + } + + pub fn is_empty(&self) -> bool { + self.index.is_empty() + } + + pub fn contains(&self, key: &CacheKey) -> bool { + self.index.contains_key(key) + } + + /// Most recently used key. + pub fn back(&self) -> Option<&CacheKey> { + if self.tail == CACHE_ORDER_NIL { + return None; + } + Some(&self.nodes[self.tail as usize].key) + } + + /// Least recently used key. + pub fn front(&self) -> Option<&CacheKey> { + if self.head == CACHE_ORDER_NIL { + return None; + } + Some(&self.nodes[self.head as usize].key) + } + + /// Append `key` as most recently used, or move it there if already present. + pub fn push_back(&mut self, key: CacheKey) { + if let Some(&node) = self.index.get(&key) { + self.unlink(node); + self.link_back(node); + return; + } + let node = self.alloc(key.clone()); + self.link_back(node); + self.index.insert(key, node); + } + + /// Append `key` at the back only if it is not already tracked. + /// + /// First-in first-out ordering must not move a key that is rewritten, so + /// this leaves an existing key exactly where it is. It still inserts a key + /// that is missing, which keeps the order consistent with an index that + /// already holds the key. + pub fn push_back_if_absent(&mut self, key: CacheKey) { + if self.index.contains_key(&key) { + return; + } + let node = self.alloc(key.clone()); + self.link_back(node); + self.index.insert(key, node); + } + + /// Insert `key` as least recently used, or move it there if already present. + pub fn push_front(&mut self, key: CacheKey) { + if let Some(&node) = self.index.get(&key) { + self.unlink(node); + self.link_front(node); + return; + } + let node = self.alloc(key.clone()); + self.link_front(node); + self.index.insert(key, node); + } + + /// Record a hit: move `key` to the back if it is present. Returns whether + /// the key was there. + pub fn touch(&mut self, key: &CacheKey) -> bool { + let Some(&node) = self.index.get(key) else { + return false; + }; + if node == self.tail { + return true; + } + self.unlink(node); + self.link_back(node); + true + } + + /// Remove `key`, returning whether it was present. + pub fn remove(&mut self, key: &CacheKey) -> bool { + let Some(node) = self.index.remove(key) else { + return false; + }; + self.unlink(node); + self.release(node); + true + } + + /// Remove and return the least recently used key. + pub fn pop_front(&mut self) -> Option { + if self.head == CACHE_ORDER_NIL { + return None; + } + let node = self.head; + let key = self.nodes[node as usize].key.clone(); + self.unlink(node); + self.index.remove(&key); + self.release(node); + Some(key) + } + + /// Keep only the keys for which `predicate` returns true. Order preserved. + pub fn retain(&mut self, mut predicate: F) + where + F: FnMut(&CacheKey) -> bool, + { + let mut cursor = self.head; + while cursor != CACHE_ORDER_NIL { + let next = self.nodes[cursor as usize].next; + if !predicate(&self.nodes[cursor as usize].key) { + let key = self.nodes[cursor as usize].key.clone(); + self.index.remove(&key); + self.unlink(cursor); + self.release(cursor); + } + cursor = next; + } + } + + pub fn clear(&mut self) { + self.nodes.clear(); + self.free.clear(); + self.index.clear(); + self.head = CACHE_ORDER_NIL; + self.tail = CACHE_ORDER_NIL; + } + + /// Iterate from most to least recently used. + pub fn iter_rev(&self) -> CacheKeyOrderRevIter<'_> { + CacheKeyOrderRevIter { + order: self, + cursor: self.tail, + } + } + + /// Iterate from least to most recently used. + pub fn iter(&self) -> CacheKeyOrderIter<'_> { + CacheKeyOrderIter { + order: self, + cursor: self.head, + } + } + + fn alloc(&mut self, key: CacheKey) -> u32 { + if let Some(index) = self.free.pop() { + let node = &mut self.nodes[index as usize]; + node.key = key; + node.prev = CACHE_ORDER_NIL; + node.next = CACHE_ORDER_NIL; + return index; + } + self.nodes.push(CacheOrderNode { + key, + prev: CACHE_ORDER_NIL, + next: CACHE_ORDER_NIL, + }); + (self.nodes.len() - 1) as u32 + } + + fn release(&mut self, node: u32) { + self.free.push(node); + } + + fn link_back(&mut self, node: u32) { + let old_tail = self.tail; + self.nodes[node as usize].prev = old_tail; + self.nodes[node as usize].next = CACHE_ORDER_NIL; + if old_tail == CACHE_ORDER_NIL { + self.head = node; + } else { + self.nodes[old_tail as usize].next = node; + } + self.tail = node; + } + + fn link_front(&mut self, node: u32) { + let old_head = self.head; + self.nodes[node as usize].next = old_head; + self.nodes[node as usize].prev = CACHE_ORDER_NIL; + if old_head == CACHE_ORDER_NIL { + self.tail = node; + } else { + self.nodes[old_head as usize].prev = node; + } + self.head = node; + } + + fn unlink(&mut self, node: u32) { + let prev = self.nodes[node as usize].prev; + let next = self.nodes[node as usize].next; + if prev == CACHE_ORDER_NIL { + self.head = next; + } else { + self.nodes[prev as usize].next = next; + } + if next == CACHE_ORDER_NIL { + self.tail = prev; + } else { + self.nodes[next as usize].prev = prev; + } + self.nodes[node as usize].prev = CACHE_ORDER_NIL; + self.nodes[node as usize].next = CACHE_ORDER_NIL; + } +} + +impl FromIterator for CacheKeyOrder { + fn from_iter>(iter: I) -> Self { + let mut order = Self::new(); + for key in iter { + order.push_back(key); + } + order + } +} + +pub struct CacheKeyOrderIter<'a> { + order: &'a CacheKeyOrder, + cursor: u32, +} + +pub struct CacheKeyOrderRevIter<'a> { + order: &'a CacheKeyOrder, + cursor: u32, +} + +impl<'a> Iterator for CacheKeyOrderRevIter<'a> { + type Item = &'a CacheKey; + + fn next(&mut self) -> Option { + if self.cursor == CACHE_ORDER_NIL { + return None; + } + let node = &self.order.nodes[self.cursor as usize]; + self.cursor = node.prev; + Some(&node.key) + } +} + +impl<'a> Iterator for CacheKeyOrderIter<'a> { + type Item = &'a CacheKey; + + fn next(&mut self) -> Option { + if self.cursor == CACHE_ORDER_NIL { + return None; + } + let node = &self.order.nodes[self.cursor as usize]; + self.cursor = node.next; + Some(&node.key) + } +} diff --git a/src/runtime/multilayer_cache.rs b/src/runtime/multilayer_cache.rs index 05b1529..0c7e8e7 100644 --- a/src/runtime/multilayer_cache.rs +++ b/src/runtime/multilayer_cache.rs @@ -232,15 +232,12 @@ struct CacheInner { memory: HashMap>, pmem: HashMap>, disk_index: HashMap, - disk_order: VecDeque, - disk_fifo_order: VecDeque, + disk_fifo_order: CacheKeyOrder, pinned: HashMap, pinned_handle_bytes: HashMap, pinned_removed_bytes: HashMap, - order: VecDeque, - memory_fifo_order: VecDeque, - pmem_order: VecDeque, - pmem_fifo_order: VecDeque, + memory_fifo_order: CacheKeyOrder, + pmem_fifo_order: CacheKeyOrder, async_writeback_queue: VecDeque, async_writeback_positions: HashMap, async_writeback_queue_bytes: u64, @@ -249,6 +246,8 @@ struct CacheInner { eviction_callback: Option, eviction_handler_enabled: bool, pending_eviction_records: VecDeque, + eviction_metric_callback: Option, + pending_eviction_metric_tiers: VecDeque, ssd_instance_only: bool, memory_replacement_policy: CacheReplacementPolicy, pmem_replacement_policy: CacheReplacementPolicy, @@ -647,15 +646,12 @@ impl MultiLayerCache { memory: HashMap::new(), pmem: HashMap::new(), disk_index: HashMap::new(), - disk_order: VecDeque::new(), - disk_fifo_order: VecDeque::new(), + disk_fifo_order: CacheKeyOrder::new(), pinned: HashMap::new(), pinned_handle_bytes: HashMap::new(), pinned_removed_bytes: HashMap::new(), - order: VecDeque::new(), - memory_fifo_order: VecDeque::new(), - pmem_order: VecDeque::new(), - pmem_fifo_order: VecDeque::new(), + memory_fifo_order: CacheKeyOrder::new(), + pmem_fifo_order: CacheKeyOrder::new(), async_writeback_queue: VecDeque::new(), async_writeback_positions: HashMap::new(), async_writeback_queue_bytes: 0, @@ -664,6 +660,8 @@ impl MultiLayerCache { eviction_callback: None, eviction_handler_enabled: true, pending_eviction_records: VecDeque::new(), + eviction_metric_callback: None, + pending_eviction_metric_tiers: VecDeque::new(), ssd_instance_only: false, memory_replacement_policy: CacheReplacementPolicy::WeightedHotnessLru, pmem_replacement_policy: CacheReplacementPolicy::WeightedHotnessLru, @@ -1013,6 +1011,25 @@ impl MultiLayerCache { inner.pending_eviction_records.clear(); } + /// Register a callback receiving the number of entries evicted from a tier. + /// + /// Independent of the eviction handler: it keeps reporting while the + /// handler is disabled, so eviction rate stays observable even when nothing + /// is consuming the evicted entries. + pub fn register_eviction_metric_callback(&self, callback: F) + where + F: Fn(CacheTier, usize) + Send + Sync + 'static, + { + let mut inner = self.inner.write().expect("cache lock poisoned"); + inner.eviction_metric_callback = Some(CacheEvictionMetricCallback::new(callback)); + } + + pub fn clear_eviction_metric_callback(&self) { + let mut inner = self.inner.write().expect("cache lock poisoned"); + inner.eviction_metric_callback = None; + inner.pending_eviction_metric_tiers.clear(); + } + pub fn set_eviction_handler_enabled(&self, enabled: bool) { let mut inner = self.inner.write().expect("cache lock poisoned"); inner.eviction_handler_enabled = enabled; @@ -1045,12 +1062,30 @@ impl MultiLayerCache { } fn drain_eviction_records(&self) { - let (callback, records) = { + let (callback, records, metric_callback, metric_tiers) = { let mut inner = self.inner.write().expect("cache lock poisoned"); let callback = inner.eviction_callback.clone(); let records = inner.pending_eviction_records.drain(..).collect::>(); - (callback, records) + let metric_callback = inner.eviction_metric_callback.clone(); + let metric_tiers = inner + .pending_eviction_metric_tiers + .drain(..) + .collect::>(); + (callback, records, metric_callback, metric_tiers) }; + // Metrics first, and ungated: they report the eviction rate whether or + // not a handler is consuming the evicted entries. + if let Some(metric_callback) = metric_callback { + let mut reported: Vec = Vec::new(); + for tier in &metric_tiers { + if reported.contains(tier) { + continue; + } + reported.push(*tier); + let count = metric_tiers.iter().filter(|entry| *entry == tier).count(); + metric_callback.call(*tier, count); + } + } if let Some(callback) = callback { for record in records { callback.call(record); @@ -1182,7 +1217,6 @@ impl MultiLayerCache { if !inner.ssd_instance_only { if let Some(value) = inner.memory.get(key).cloned() { inner.stats.memory_hits += 1; - inner.touch_key(key); inner.record_hit(key, value.len()); inner.record_get_latency(started); inner.record_read_through_latency(started); @@ -1193,7 +1227,6 @@ impl MultiLayerCache { } if let Some(value) = inner.pmem.get(key).cloned() { inner.stats.pmem_hits = inner.stats.pmem_hits.saturating_add(1); - inner.touch_key(key); inner.record_hit(key, value.len()); let decoded = value.to_vec(); if !inner.put_memory(key.clone(), decoded.clone()) { @@ -1259,8 +1292,6 @@ impl MultiLayerCache { let mut ssd_candidates = Vec::new(); let mut needs_eviction_drain = false; { - let mut memory_touches = Vec::new(); - let mut pmem_touches = Vec::new(); let mut disk_touches = Vec::new(); let mut inner = self.inner.write().expect("cache lock poisoned"); if !inner.started { @@ -1272,7 +1303,6 @@ impl MultiLayerCache { if let Some(value) = inner.memory.get(key).cloned() { inner.stats.memory_hits = inner.stats.memory_hits.saturating_add(1); inner.record_hit_metadata(key, value.len()); - memory_touches.push(key.clone()); if inner.disk_index.contains_key(key) { disk_touches.push(key.clone()); } @@ -1284,7 +1314,6 @@ impl MultiLayerCache { if let Some(value) = inner.pmem.get(key).cloned() { inner.stats.pmem_hits = inner.stats.pmem_hits.saturating_add(1); inner.record_hit_metadata(key, value.len()); - pmem_touches.push(key.clone()); if inner.disk_index.contains_key(key) { disk_touches.push(key.clone()); } @@ -1303,7 +1332,6 @@ impl MultiLayerCache { } ssd_candidates.push((index, key.clone(), started)); } - inner.touch_hit_queues_batch(&disk_touches, &memory_touches, &pmem_touches); } if ssd_candidates.is_empty() { @@ -1386,7 +1414,6 @@ impl MultiLayerCache { } } } - inner.touch_hit_queues_batch(&disk_touches, &[], &[]); } if needs_eviction_drain { self.drain_eviction_records(); @@ -1403,7 +1430,6 @@ impl MultiLayerCache { let value = inner.memory.get(key).cloned(); if value.is_some() { inner.stats.memory_hits += 1; - inner.touch_key(key); inner.record_hit( key, value.as_ref().map(|bytes| bytes.len()).unwrap_or_default(), @@ -1433,7 +1459,6 @@ impl MultiLayerCache { inner.stats.zero_copy_handle_hits = inner.stats.zero_copy_handle_hits.saturating_add(1); inner.stats.memory_hits = inner.stats.memory_hits.saturating_add(1); - inner.touch_key(key); inner.record_hit(key, value.len()); inner.refresh_pin_stats(); inner.record_get_latency(started); @@ -1452,7 +1477,6 @@ impl MultiLayerCache { inner.stats.zero_copy_handle_hits = inner.stats.zero_copy_handle_hits.saturating_add(1); inner.stats.pmem_hits = inner.stats.pmem_hits.saturating_add(1); - inner.touch_key(key); inner.record_hit(key, value.len()); inner.refresh_pin_stats(); inner.record_get_latency(started); @@ -2651,12 +2675,10 @@ impl MultiLayerCache { removed_pinned_bytes = removed_pinned_bytes.max(value.len()); inner.memory_bytes = inner.memory_bytes.saturating_sub(value.len()); } - inner.order.retain(|candidate| candidate != key); if let Some(value) = inner.pmem.remove(key) { removed_pinned_bytes = removed_pinned_bytes.max(value.len()); inner.pmem_bytes = inner.pmem_bytes.saturating_sub(value.len()); } - inner.pmem_order.retain(|candidate| candidate != key); if key_pinned && removed_pinned_bytes > 0 { inner .pinned_removed_bytes @@ -2726,9 +2748,7 @@ impl MultiLayerCache { if enabled { inner.memory.clear(); inner.pmem.clear(); - inner.order.clear(); inner.memory_fifo_order.clear(); - inner.pmem_order.clear(); inner.pmem_fifo_order.clear(); inner.memory_bytes = 0; inner.pmem_bytes = 0; @@ -4795,11 +4815,8 @@ impl CacheInner { self.write_ssd_block(&key, &block)?; if let Some(old_len) = self.disk_index.insert(key.clone(), block_len as u64) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); - self.disk_order.retain(|candidate| candidate != &key); - self.disk_fifo_order.retain(|candidate| candidate != &key); } - self.disk_order.push_back(key.clone()); - self.disk_fifo_order.push_back(key.clone()); + self.disk_fifo_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len as u64); self.record_metadata( &key, @@ -5011,13 +5028,9 @@ impl CacheInner { .collect::>(); if staged_keys.len() > SET_MEMBERSHIP_THRESHOLD { let staged_key_set = staged_keys.iter().cloned().collect::>(); - self.disk_order - .retain(|candidate| !staged_key_set.contains(candidate)); self.disk_fifo_order .retain(|candidate| !staged_key_set.contains(candidate)); } else { - self.disk_order - .retain(|candidate| !staged_keys.contains(candidate)); self.disk_fifo_order .retain(|candidate| !staged_keys.contains(candidate)); } @@ -5026,8 +5039,7 @@ impl CacheInner { if let Some(old_len) = self.disk_index.insert(entry.key.clone(), entry.block_len) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } - self.disk_order.push_back(entry.key.clone()); - self.disk_fifo_order.push_back(entry.key.clone()); + self.disk_fifo_order.push_back_if_absent(entry.key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(entry.block_len); self.record_metadata( &entry.key, @@ -5103,11 +5115,8 @@ impl CacheInner { }; if let Some(old_len) = self.disk_index.insert(key.clone(), block_len) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); - self.disk_order.retain(|candidate| candidate != &key); - self.disk_fifo_order.retain(|candidate| candidate != &key); } - self.disk_order.push_back(key.clone()); - self.disk_fifo_order.push_back(key.clone()); + self.disk_fifo_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len); self.record_metadata( &key, @@ -5168,11 +5177,8 @@ impl CacheInner { self.write_ssd_block(&key, &block)?; if let Some(old_len) = self.disk_index.insert(key.clone(), block_len as u64) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); - self.disk_order.retain(|candidate| candidate != &key); - self.disk_fifo_order.retain(|candidate| candidate != &key); } - self.disk_order.push_back(key.clone()); - self.disk_fifo_order.push_back(key.clone()); + self.disk_fifo_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len as u64); self.record_metadata( &key, @@ -5202,8 +5208,7 @@ impl CacheInner { self.pinned_removed_bytes.insert(key.clone(), value.len()); } } - self.order.retain(|candidate| candidate != key); - self.memory_fifo_order.retain(|candidate| candidate != key); + self.memory_fifo_order.remove(key); } CacheTier::Pmem => { if let Some(value) = self.pmem.remove(key) { @@ -5212,8 +5217,7 @@ impl CacheInner { self.pinned_removed_bytes.insert(key.clone(), value.len()); } } - self.pmem_order.retain(|candidate| candidate != key); - self.pmem_fifo_order.retain(|candidate| candidate != key); + self.pmem_fifo_order.remove(key); self.persist_pmem_delete(key)?; } CacheTier::Ssd => { @@ -5221,8 +5225,7 @@ impl CacheInner { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } self.delete_ssd_block(key)?; - self.disk_order.retain(|candidate| candidate != key); - self.disk_fifo_order.retain(|candidate| candidate != key); + self.disk_fifo_order.remove(key); self.stats.disk_bytes = self.ssd_bytes; self.append_disk_manifest_delete(key)?; } @@ -5284,7 +5287,6 @@ impl CacheInner { .saturating_sub(current.len()) .saturating_add(new_value.len()); *current = Arc::clone(&new_value); - self.touch_key(key); self.stats.memory_bytes = self.memory_bytes as u64; self.refresh_pin_stats(); Ok(()) @@ -5301,7 +5303,6 @@ impl CacheInner { .saturating_sub(current.len()) .saturating_add(new_value.len()); *current = Arc::clone(&new_value); - self.touch_key(key); self.stats.pmem_bytes = self.pmem_bytes as u64; self.refresh_pin_stats(); Ok(()) @@ -5325,23 +5326,20 @@ impl CacheInner { .remove(key) .unwrap_or(existing_block.len() as u64); self.ssd_bytes = self.ssd_bytes.saturating_sub(indexed_old_len); - self.disk_order.retain(|candidate| candidate != key); - self.disk_fifo_order.retain(|candidate| candidate != key); + self.disk_fifo_order.remove(key); self.evict_ssd_for(block.len() as u64); if self.ssd_bytes.saturating_add(block.len() as u64) > self.ssd_capacity_bytes as u64 { self.disk_index.insert(key.clone(), indexed_old_len); - self.disk_order.push_back(key.clone()); - self.disk_fifo_order.push_back(key.clone()); + self.disk_fifo_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(indexed_old_len); self.stats.disk_bytes = self.ssd_bytes; return Err(CacheError::CapacityExceeded); } self.write_ssd_block(key, &block)?; self.disk_index.insert(key.clone(), block.len() as u64); - self.disk_order.push_back(key.clone()); - self.disk_fifo_order.push_back(key.clone()); + self.disk_fifo_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block.len() as u64); self.record_metadata( key, @@ -5411,11 +5409,9 @@ impl MultiLayerCache { inner.pmem_bytes = inner.pmem_bytes.saturating_sub(value.len()); } } - inner.order.retain(|key| key.shard_id != shard_id); inner .memory_fifo_order .retain(|key| key.shard_id != shard_id); - inner.pmem_order.retain(|key| key.shard_id != shard_id); inner.pmem_fifo_order.retain(|key| key.shard_id != shard_id); let disk_keys = inner .disk_index @@ -5429,7 +5425,6 @@ impl MultiLayerCache { disk_bytes_before.saturating_add(inner.disk_index.remove(key).unwrap_or_default()); } let _ = inner.delete_ssd_blocks(&disk_keys); - inner.disk_order.retain(|key| key.shard_id != shard_id); inner.disk_fifo_order.retain(|key| key.shard_id != shard_id); inner.metadata.retain(|key, _| key.shard_id != shard_id); inner.pinned.retain(|key, _| key.shard_id != shard_id); @@ -5496,21 +5491,12 @@ impl MultiLayerCache { inner.metadata.remove(key); } let _ = inner.delete_ssd_blocks(&disk_delete_keys); - inner - .order - .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); inner .memory_fifo_order .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); - inner - .pmem_order - .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); inner .pmem_fifo_order .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); - inner - .disk_order - .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); inner .disk_fifo_order .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); @@ -5573,21 +5559,12 @@ impl MultiLayerCache { inner.metadata.remove(key); } let _ = inner.delete_ssd_blocks(&disk_delete_keys); - inner.order.retain(|key| { - !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) - }); inner.memory_fifo_order.retain(|key| { !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) }); - inner.pmem_order.retain(|key| { - !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) - }); inner.pmem_fifo_order.retain(|key| { !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) }); - inner.disk_order.retain(|key| { - !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) - }); inner.disk_fifo_order.retain(|key| { !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) }); @@ -5825,8 +5802,6 @@ impl MultiLayerCache { let mut inner = self.inner.write().expect("cache lock poisoned"); inner.memory.clear(); inner.pmem.clear(); - inner.order.clear(); - inner.pmem_order.clear(); inner.memory_bytes = 0; inner.pmem_bytes = 0; inner.stats.memory_bytes = 0; diff --git a/src/runtime/replacement.rs b/src/runtime/replacement.rs index 2f5fce1..20d6d56 100644 --- a/src/runtime/replacement.rs +++ b/src/runtime/replacement.rs @@ -261,18 +261,182 @@ impl CacheReplacementPolicy { pub type CacheKeyType = String; +/// Sentinel for "no node" in the intrusive key lists below. +const KEY_LIST_NIL: u32 = u32::MAX; + +/// One slot in a [`KeyArena`]. A node is linked into at most one [`KeyList`] at +/// a time and carries its key, so eviction can walk from a list end back into +/// the owning index. +#[derive(Debug, Clone)] +struct KeyListNode { + key: CacheKeyType, + prev: u32, + next: u32, +} + +/// Backing storage for [`KeyList`]. Released slots are recycled, so a policy +/// that churns keys does not grow the arena without bound. +#[derive(Debug, Clone, Default)] +struct KeyArena { + nodes: Vec, + free: Vec, +} + +/// An intrusive doubly-linked list of keys held in a [`KeyArena`]. +/// +/// Unlike a `VecDeque` of keys, this unlinks or re-fronts an arbitrary element +/// in O(1), which is what keeps policy lookups and deletes independent of how +/// many entries are cached. `used` is a byte counter for policies that budget +/// their lists by size; policies that only care about order leave it at zero. +#[derive(Debug, Clone)] +struct KeyList { + head: u32, + tail: u32, + used: usize, + len: usize, +} + +impl KeyList { + fn new() -> Self { + Self { + head: KEY_LIST_NIL, + tail: KEY_LIST_NIL, + used: 0, + len: 0, + } + } + + fn clear(&mut self) { + self.head = KEY_LIST_NIL; + self.tail = KEY_LIST_NIL; + self.used = 0; + self.len = 0; + } + + fn front(&self) -> Option { + (self.head != KEY_LIST_NIL).then_some(self.head) + } + + fn back(&self) -> Option { + (self.tail != KEY_LIST_NIL).then_some(self.tail) + } +} + +impl Default for KeyList { + fn default() -> Self { + Self::new() + } +} + +impl KeyArena { + fn new() -> Self { + Self::default() + } + + fn alloc(&mut self, key: &str) -> u32 { + if let Some(index) = self.free.pop() { + let node = &mut self.nodes[index as usize]; + node.key.clear(); + node.key.push_str(key); + node.prev = KEY_LIST_NIL; + node.next = KEY_LIST_NIL; + return index; + } + self.nodes.push(KeyListNode { + key: key.to_string(), + prev: KEY_LIST_NIL, + next: KEY_LIST_NIL, + }); + (self.nodes.len() - 1) as u32 + } + + fn release(&mut self, node: u32) { + let slot = &mut self.nodes[node as usize]; + slot.key.clear(); + slot.prev = KEY_LIST_NIL; + slot.next = KEY_LIST_NIL; + self.free.push(node); + } + + fn key(&self, node: u32) -> &str { + &self.nodes[node as usize].key + } + + fn clear(&mut self) { + self.nodes.clear(); + self.free.clear(); + } + + fn link_front(&mut self, list: &mut KeyList, node: u32) { + let old_head = list.head; + self.nodes[node as usize].prev = KEY_LIST_NIL; + self.nodes[node as usize].next = old_head; + if old_head == KEY_LIST_NIL { + list.tail = node; + } else { + self.nodes[old_head as usize].prev = node; + } + list.head = node; + list.len = list.len.saturating_add(1); + } + + fn link_back(&mut self, list: &mut KeyList, node: u32) { + let old_tail = list.tail; + self.nodes[node as usize].next = KEY_LIST_NIL; + self.nodes[node as usize].prev = old_tail; + if old_tail == KEY_LIST_NIL { + list.head = node; + } else { + self.nodes[old_tail as usize].next = node; + } + list.tail = node; + list.len = list.len.saturating_add(1); + } + + fn unlink(&mut self, list: &mut KeyList, node: u32) { + let prev = self.nodes[node as usize].prev; + let next = self.nodes[node as usize].next; + if prev == KEY_LIST_NIL { + list.head = next; + } else { + self.nodes[prev as usize].next = next; + } + if next == KEY_LIST_NIL { + list.tail = prev; + } else { + self.nodes[next as usize].prev = prev; + } + self.nodes[node as usize].prev = KEY_LIST_NIL; + self.nodes[node as usize].next = KEY_LIST_NIL; + list.len = list.len.saturating_sub(1); + } + + /// Collect up to `size` keys walking from the tail toward the head. + fn collect_from_tail(&self, list: &KeyList, size: usize) -> Vec { + let mut collected = Vec::new(); + let mut cursor = list.tail; + while cursor != KEY_LIST_NIL && collected.len() < size { + collected.push(self.nodes[cursor as usize].key.clone()); + cursor = self.nodes[cursor as usize].prev; + } + collected + } +} + #[derive(Debug, Clone)] pub struct BaseLRUList { capacity: usize, - list: VecDeque, - index: HashMap, + arena: KeyArena, + list: KeyList, + index: HashMap, } impl BaseLRUList { pub fn new(capacity: usize) -> Self { Self { capacity, - list: VecDeque::new(), + arena: KeyArena::new(), + list: KeyList::new(), index: HashMap::new(), } } @@ -290,51 +454,64 @@ impl BaseLRUList { } pub fn put(&mut self, key: CacheKeyType) { - self.delete(&key); - self.list.push_front(key.clone()); - self.index.insert(key, ()); + if let Some(&node) = self.index.get(&key) { + self.arena.unlink(&mut self.list, node); + self.arena.link_front(&mut self.list, node); + return; + } + let node = self.arena.alloc(&key); + self.arena.link_front(&mut self.list, node); + self.index.insert(key, node); } pub fn get(&mut self, key: &str) -> bool { - if !self.index.contains_key(key) { + let Some(&node) = self.index.get(key) else { return false; - } - let key = key.to_string(); - self.list.retain(|candidate| candidate != &key); - self.list.push_front(key); + }; + self.arena.unlink(&mut self.list, node); + self.arena.link_front(&mut self.list, node); true } pub fn delete(&mut self, key: &str) -> bool { - if self.index.remove(key).is_some() { - self.list.retain(|candidate| candidate != key); - return true; - } - false + let Some(node) = self.index.remove(key) else { + return false; + }; + self.arena.unlink(&mut self.list, node); + self.arena.release(node); + true } pub fn get_tail(&self, size: usize) -> Vec { - self.list.iter().rev().take(size).cloned().collect() + self.arena.collect_from_tail(&self.list, size) } pub fn evict(&mut self) -> Vec { let mut evicted = Vec::new(); while self.index.len() > self.capacity { - evicted.extend(self.evict_one()); + let popped = self.evict_one(); + if popped.is_empty() { + break; + } + evicted.extend(popped); } evicted } pub fn evict_one(&mut self) -> Vec { - if let Some(key) = self.list.pop_back() { - self.index.remove(&key); - return vec![key]; - } - Vec::new() + let Some(node) = self.list.back() else { + return Vec::new(); + }; + let key = self.arena.key(node).to_string(); + self.arena.unlink(&mut self.list, node); + self.arena.release(node); + self.index.remove(&key); + vec![key] } pub fn reset(&mut self) { self.list.clear(); + self.arena.clear(); self.index.clear(); } } @@ -1037,10 +1214,17 @@ impl ReplacementPolicyBase { pub type ReplacementPolicy = ReplacementPolicyBase; +#[derive(Debug)] +struct FifoEntry { + buffer: CacheBuffer, + node: u32, +} + pub struct ReplacementFIFO { base: ReplacementPolicyBase, - index: HashMap, - queue: VecDeque, + index: HashMap, + arena: KeyArena, + queue: KeyList, mem_eviction_func: Option, } @@ -1049,7 +1233,8 @@ impl ReplacementFIFO { Self { base: ReplacementPolicyBase::new(capacity), index: HashMap::new(), - queue: VecDeque::new(), + arena: KeyArena::new(), + queue: KeyList::new(), mem_eviction_func: None, } } @@ -1061,11 +1246,18 @@ impl ReplacementFIFO { pub fn reset(&mut self) -> Result<(), CacheError> { self.index.clear(); self.queue.clear(); + self.arena.clear(); self.base.used = 0; self.base.initialized = false; Ok(()) } + /// Track `buffer`, evicting from the front of the queue if that pushes the + /// policy over capacity. + /// + /// Overwriting a key that is already tracked keeps its original queue + /// position: first-in-first-out orders by when a key first entered the + /// cache, not by when it was last written. Only the byte accounting moves. pub fn put(&mut self, buffer: CacheBuffer) -> Vec { if !self.base.initialized { return Vec::new(); @@ -1074,13 +1266,21 @@ impl ReplacementFIFO { if key.is_empty() { return Vec::new(); } - if let Some(old) = self.index.remove(&key) { - self.base.used = self.base.used.saturating_sub(cache_buffer_space(&old)); - self.queue.retain(|candidate| candidate != &key); + let space = cache_buffer_space(&buffer); + if let Some(entry) = self.index.get_mut(&key) { + let old_space = cache_buffer_space(&entry.buffer); + entry.buffer = buffer; + self.base.used = self + .base + .used + .saturating_sub(old_space) + .saturating_add(space); + } else { + let node = self.arena.alloc(&key); + self.arena.link_back(&mut self.queue, node); + self.base.used = self.base.used.saturating_add(space); + self.index.insert(key, FifoEntry { buffer, node }); } - self.base.used = self.base.used.saturating_add(cache_buffer_space(&buffer)); - self.queue.push_back(key.clone()); - self.index.insert(key, buffer); self.evict_to_capacity() } @@ -1090,19 +1290,28 @@ impl ReplacementFIFO { old_data: &[u8], buffer: CacheBuffer, ) -> Result<(), CacheError> { - let old = self.index.get(key).ok_or(CacheError::NotFound)?; - if old.data() != old_data { + let entry = self.index.get_mut(key).ok_or(CacheError::NotFound)?; + if entry.buffer.data() != old_data { return Err(CacheError::ReplaceMismatch); } - self.base.used = self.base.used.saturating_sub(cache_buffer_space(old)); - self.base.used = self.base.used.saturating_add(cache_buffer_space(&buffer)); - self.index.insert(key.to_string(), buffer); + let old_space = cache_buffer_space(&entry.buffer); + let new_space = cache_buffer_space(&buffer); + entry.buffer = buffer; + self.base.used = self + .base + .used + .saturating_sub(old_space) + .saturating_add(new_space); let _ = self.evict_to_capacity(); Ok(()) } + /// A read never changes eviction order under this policy, so `get` and + /// `peek` are the same lookup. pub fn get(&self, key: &str) -> Option { - self.index.get(key).map(clone_cache_buffer) + self.index + .get(key) + .map(|entry| clone_cache_buffer(&entry.buffer)) } pub fn peek(&self, key: &str) -> Option { @@ -1110,10 +1319,14 @@ impl ReplacementFIFO { } pub fn delete(&mut self, key: &str) -> Option { - let buffer = self.index.remove(key)?; - self.base.used = self.base.used.saturating_sub(cache_buffer_space(&buffer)); - self.queue.retain(|candidate| candidate != key); - Some(buffer) + let entry = self.index.remove(key)?; + self.base.used = self + .base + .used + .saturating_sub(cache_buffer_space(&entry.buffer)); + self.arena.unlink(&mut self.queue, entry.node); + self.arena.release(entry.node); + Some(entry.buffer) } pub fn get_capacity(&self) -> usize { @@ -1137,6 +1350,12 @@ impl ReplacementFIFO { self.index.len() } + /// Entries currently queued. Equal to [`ReplacementFIFO::get_item_num`]; + /// the queue holds no tombstones for deleted keys. + pub fn queue_len(&self) -> usize { + self.queue.len + } + pub fn register_mem_eviction_handler(&mut self, func: F) where F: Fn(CacheBuffer) + Send + Sync + 'static, @@ -1147,17 +1366,23 @@ impl ReplacementFIFO { fn evict_to_capacity(&mut self) -> Vec { let mut evicted = Vec::new(); while self.base.used > self.base.capacity { - let Some(key) = self.queue.pop_front() else { + let Some(node) = self.queue.front() else { break; }; - let Some(buffer) = self.index.remove(&key) else { + let key = self.arena.key(node).to_string(); + self.arena.unlink(&mut self.queue, node); + self.arena.release(node); + let Some(entry) = self.index.remove(&key) else { continue; }; - self.base.used = self.base.used.saturating_sub(cache_buffer_space(&buffer)); + self.base.used = self + .base + .used + .saturating_sub(cache_buffer_space(&entry.buffer)); if let Some(handler) = &self.mem_eviction_func { - handler(clone_cache_buffer(&buffer)); + handler(clone_cache_buffer(&entry.buffer)); } - evicted.push(buffer); + evicted.push(entry.buffer); } evicted } @@ -1226,31 +1451,131 @@ impl ReplacementFIFO { } } +/// Default number of hash-partitioned SLRU segments. +pub const SLRU_DEFAULT_NUM_SEGMENTS: usize = 256; +/// Share of a segment byte budget the hot list is allowed to hold, in percent. +pub const SLRU_DEFAULT_HOT_LRU_PCT: u32 = 20; +/// Share of a segment byte budget the warm list is allowed to hold, in percent. +pub const SLRU_DEFAULT_WARM_LRU_PCT: u32 = 40; + +const SLRU_LIST_COUNT: usize = 3; +/// Upper bound on the configurable segment count, so that rounding the request +/// up to a power of two can never overflow. +const SLRU_MAX_NUM_SEGMENTS: usize = 1 << 20; + +/// Resolve the effective segment count: the request is rounded up to a power of +/// two so that segment selection can mask instead of divide, and a capacity +/// smaller than the segment count collapses to a single segment (otherwise each +/// segment would get a zero byte budget and evict every insert immediately). +fn resolve_slru_num_segments(capacity: usize, requested: usize) -> usize { + let requested = requested.clamp(1, SLRU_MAX_NUM_SEGMENTS).next_power_of_two(); + if capacity < requested { + 1 + } else { + requested + } +} + #[derive(Debug)] struct SlruEntry { buffer: CacheBuffer, flag: u16, lru: u16, + segment: u32, + node: u32, } +/// A hash-partitioned shard: three LRU lists plus the shard byte accounting. +#[derive(Debug)] +struct SlruSegment { + lists: [KeyList; SLRU_LIST_COUNT], + used: usize, +} + +impl SlruSegment { + fn new() -> Self { + Self { + lists: [KeyList::new(), KeyList::new(), KeyList::new()], + used: 0, + } + } +} + +/// What the tail of a list should become, decided while the index entry is +/// borrowed and applied afterwards against the lists. +enum SlruTailAction { + /// The list node has no index entry left; drop the node. + Orphan, + /// Move the node to the front of `.0`, whose payload occupies `.1` bytes. + MoveTo(u16, usize), + /// Evict the entry outright. + Evict, +} + +/// Outcome of one background-maintainer step over a cold list tail. +enum SlruMaintainerStep { + /// Nothing left to do in this list. + Stop, + /// The tail was reshuffled into another list. + Moved, + /// The tail was evicted. + Evicted(CacheBuffer), +} + +/// Segmented LRU replacement policy. +/// +/// The cache is hash-partitioned into [`ReplacementSLRU::num_segments`] shards, +/// each holding an independent hot/warm/cold LRU triple with its own byte +/// budget of `capacity / num_segments`. Inserts, lookups and eviction only ever +/// touch the shard owning the key, so eviction scans a small shard-local list +/// instead of one global list, and no single hot list can consume the whole +/// budget. +/// +/// A background maintainer keeps each shard's hot and warm lists within their +/// configured share of the shard budget ([`SLRU_DEFAULT_HOT_LRU_PCT`] and +/// [`SLRU_DEFAULT_WARM_LRU_PCT`]), demoting entries that were not touched twice +/// and promoting the ones that were. Because this policy is driven through +/// `&mut self` rather than by its own thread, the maintainer runs as an +/// explicit pass: [`ReplacementSLRU::run_lru_maintainer_pass`] sweeps every +/// shard, and each `put` maintains just the shard it touched. Passes are +/// disabled by [`ReplacementSLRU::test_config_lru_maintainer`]. pub struct ReplacementSLRU { base: ReplacementPolicyBase, index: HashMap, - hot: VecDeque, - warm: VecDeque, - cold: VecDeque, + segments: Vec, + arena: KeyArena, + num_segments: usize, + bytes_each_segment: usize, + hot_lru_pct: u32, + warm_lru_pct: u32, lru_maintainer_enabled: bool, mem_eviction_func: Option, } impl ReplacementSLRU { pub fn new(capacity: usize) -> Self { + Self::with_num_segments(capacity, SLRU_DEFAULT_NUM_SEGMENTS) + } + + /// Build a policy with an explicit shard count. The request is rounded up to + /// a power of two, and collapses to a single shard when `capacity` is + /// smaller than the shard count. + pub fn with_num_segments(capacity: usize, num_segments: usize) -> Self { + let base = ReplacementPolicyBase::new(capacity); + let num_segments = resolve_slru_num_segments(capacity, num_segments); + let mut segments = Vec::with_capacity(num_segments); + for _ in 0..num_segments { + segments.push(SlruSegment::new()); + } Self { - base: ReplacementPolicyBase::new(capacity), + base, index: HashMap::new(), - hot: VecDeque::new(), - warm: VecDeque::new(), - cold: VecDeque::new(), + segments, + arena: KeyArena::new(), + num_segments, + bytes_each_segment: capacity / num_segments, + hot_lru_pct: SLRU_DEFAULT_HOT_LRU_PCT, + warm_lru_pct: SLRU_DEFAULT_WARM_LRU_PCT, lru_maintainer_enabled: true, mem_eviction_func: None, } @@ -1262,9 +1587,13 @@ impl ReplacementSLRU { pub fn reset(&mut self) -> Result<(), CacheError> { self.index.clear(); - self.hot.clear(); - self.warm.clear(); - self.cold.clear(); + for segment in &mut self.segments { + segment.used = 0; + for list in &mut segment.lists { + list.clear(); + } + } + self.arena.clear(); self.base.used = 0; self.base.initialized = false; Ok(()) @@ -1278,24 +1607,32 @@ impl ReplacementSLRU { if key.is_empty() { return Vec::new(); } + let segment = self.pick_segment(&key); if let Some(old) = self.index.remove(&key) { - self.base.used = self - .base - .used - .saturating_sub(cache_buffer_space(&old.buffer)); - self.remove_from_lists(&key); + let space = cache_buffer_space(&old.buffer); + self.detach(old.segment, old.lru, old.node, space); + self.release_node(old.node); } - self.base.used = self.base.used.saturating_add(cache_buffer_space(&buffer)); - self.hot.push_front(key.clone()); + let space = cache_buffer_space(&buffer); + let node = self.alloc_node(&key); + self.attach_front(segment, HOT_LRU, node, space); self.index.insert( key, SlruEntry { buffer, flag: BUFFER_INIT, lru: HOT_LRU, + segment, + node, }, ); - self.evict_to_capacity() + + let mut evicted = Vec::new(); + if self.lru_maintainer_enabled { + self.maintain_segment(segment, &mut evicted); + } + self.evict_segment(segment, &mut evicted); + evicted } pub fn update_cache_buffer( @@ -1308,21 +1645,36 @@ impl ReplacementSLRU { if entry.buffer.data() != old_data { return Err(CacheError::ReplaceMismatch); } - self.base.used = self - .base - .used - .saturating_sub(cache_buffer_space(&entry.buffer)); - self.base.used = self.base.used.saturating_add(cache_buffer_space(&buffer)); + let old_space = cache_buffer_space(&entry.buffer); + let new_space = cache_buffer_space(&buffer); entry.buffer = buffer; - let _ = self.evict_to_capacity(); + let segment = entry.segment; + let lru = entry.lru; + + let shard = &mut self.segments[segment as usize]; + let list = &mut shard.lists[lru as usize]; + list.used = list.used.saturating_sub(old_space).saturating_add(new_space); + shard.used = shard.used.saturating_sub(old_space).saturating_add(new_space); + + let mut evicted = Vec::new(); + self.evict_segment(segment, &mut evicted); Ok(()) } + /// Look up `key` and record the access. + /// + /// Only the access flag moves: an entry is not re-fronted in its list on a + /// read. List position is decided by the maintainer and by eviction, which + /// is what keeps this a segmented policy rather than a plain LRU — reading + /// a key once does not jump it ahead of one that was read twice. pub fn get(&mut self, key: &str) -> Option { - self.touch(key); - self.index - .get(key) - .map(|entry| clone_cache_buffer(&entry.buffer)) + let entry = self.index.get_mut(key)?; + entry.flag = if entry.flag == BUFFER_INIT { + BUFFER_FETCHED + } else { + BUFFER_ACTIVE + }; + Some(clone_cache_buffer(&entry.buffer)) } pub fn peek(&self, key: &str) -> Option { @@ -1333,11 +1685,9 @@ impl ReplacementSLRU { pub fn delete(&mut self, key: &str) -> Option { let entry = self.index.remove(key)?; - self.base.used = self - .base - .used - .saturating_sub(cache_buffer_space(&entry.buffer)); - self.remove_from_lists(key); + let space = cache_buffer_space(&entry.buffer); + self.detach(entry.segment, entry.lru, entry.node, space); + self.release_node(entry.node); Some(entry.buffer) } @@ -1347,21 +1697,98 @@ impl ReplacementSLRU { pub fn set_capacity(&mut self, capacity: usize) { self.base.set_capacity(capacity); - let _ = self.evict_to_capacity(); + self.update_segment_byte_limit(); + let mut evicted = Vec::new(); + for segment in 0..self.num_segments as u32 { + self.evict_segment(segment, &mut evicted); + } } pub fn get_used_space(&self) -> usize { - self.base.get_used_space() + self.segments.iter().map(|segment| segment.used).sum() } pub fn get_free_space(&self) -> usize { - self.base.get_free_space() + self.get_capacity().saturating_sub(self.get_used_space()) } pub fn get_item_num(&self) -> usize { self.index.len() } + /// Number of hash-partitioned shards. + pub fn num_segments(&self) -> usize { + self.num_segments + } + + /// Byte budget of a single shard, i.e. `capacity / num_segments`. + pub fn segment_byte_limit(&self) -> usize { + self.bytes_each_segment + } + + /// Bytes currently held by one shard across its hot, warm and cold lists. + pub fn segment_used_size(&self, segment_id: usize) -> usize { + self.segments + .get(segment_id) + .map(|segment| segment.used) + .unwrap_or(0) + } + + /// Bytes currently held by one hot/warm/cold list of one shard. + pub fn list_used_size(&self, segment_id: usize, lru: u16) -> usize { + self.segments + .get(segment_id) + .and_then(|segment| segment.lists.get(lru as usize)) + .map(|list| list.used) + .unwrap_or(0) + } + + /// Entry count of one hot/warm/cold list of one shard. + pub fn list_item_num(&self, segment_id: usize, lru: u16) -> usize { + self.segments + .get(segment_id) + .and_then(|segment| segment.lists.get(lru as usize)) + .map(|list| list.len) + .unwrap_or(0) + } + + /// Shard that owns `key`. + pub fn segment_for_key(&self, key: &str) -> usize { + self.pick_segment(key) as usize + } + + pub fn hot_lru_pct(&self) -> u32 { + self.hot_lru_pct + } + + /// Set the share of a shard budget the hot list may hold, in percent. + pub fn set_hot_lru_pct(&mut self, pct: u32) { + self.hot_lru_pct = pct.min(100); + } + + pub fn warm_lru_pct(&self) -> u32 { + self.warm_lru_pct + } + + /// Set the share of a shard budget the warm list may hold, in percent. + pub fn set_warm_lru_pct(&mut self, pct: u32) { + self.warm_lru_pct = pct.min(100); + } + + /// Run one maintainer sweep across every shard, the work a background + /// maintainer thread would do on its interval. Returns the buffers evicted + /// by the sweep. Does nothing while the maintainer is disabled. + pub fn run_lru_maintainer_pass(&mut self) -> Vec { + let mut evicted = Vec::new(); + if !self.base.initialized || !self.lru_maintainer_enabled { + return evicted; + } + for segment in 0..self.num_segments as u32 { + self.maintain_segment(segment, &mut evicted); + } + evicted + } + pub fn register_mem_eviction_handler(&mut self, func: F) where F: Fn(CacheBuffer) + Send + Sync + 'static, @@ -1388,142 +1815,327 @@ impl ReplacementSLRU { pub fn test_notify_maintainer_move_complete(&self) {} - fn touch(&mut self, key: &str) { - let Some(lru) = self.index.get_mut(key).map(|entry| { - entry.flag = if entry.flag == BUFFER_INIT { - BUFFER_FETCHED - } else { - BUFFER_ACTIVE - }; - entry.lru - }) else { - return; - }; - self.move_to_front(key, lru); + fn pick_segment(&self, key: &str) -> u32 { + if self.num_segments <= 1 { + return 0; + } + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + (hasher.finish() & (self.num_segments as u64 - 1)) as u32 } - fn evict_to_capacity(&mut self) -> Vec { - let mut evicted = Vec::new(); + fn update_segment_byte_limit(&mut self) { + self.bytes_each_segment = self.base.get_capacity() / self.num_segments; + } + + fn alloc_node(&mut self, key: &str) -> u32 { + self.arena.alloc(key) + } + + fn release_node(&mut self, node: u32) { + self.arena.release(node); + } + + fn attach_front(&mut self, segment: u32, lru: u16, node: u32, space: usize) { + let shard = &mut self.segments[segment as usize]; + self.arena.link_front(&mut shard.lists[lru as usize], node); + let list = &mut shard.lists[lru as usize]; + list.used = list.used.saturating_add(space); + shard.used = shard.used.saturating_add(space); + } + + fn detach(&mut self, segment: u32, lru: u16, node: u32, space: usize) { + let shard = &mut self.segments[segment as usize]; + self.arena.unlink(&mut shard.lists[lru as usize], node); + let list = &mut shard.lists[lru as usize]; + list.used = list.used.saturating_sub(space); + shard.used = shard.used.saturating_sub(space); + } + + /// Move `node` to the front of `to`. `from == to` re-fronts it in place. + /// The shard total is unchanged: the payload stays in the same shard. + fn move_node(&mut self, segment: u32, from: u16, to: u16, node: u32, space: usize) { + let shard = &mut self.segments[segment as usize]; + self.arena.unlink(&mut shard.lists[from as usize], node); + shard.lists[from as usize].used = shard.lists[from as usize].used.saturating_sub(space); + self.arena.link_front(&mut shard.lists[to as usize], node); + shard.lists[to as usize].used = shard.lists[to as usize].used.saturating_add(space); + } + + fn list_tail(&self, segment: u32, lru: u16) -> Option { + let tail = self.segments[segment as usize].lists[lru as usize].tail; + if tail == KEY_LIST_NIL { + None + } else { + Some(tail) + } + } + + fn node_key(&self, node: u32) -> String { + self.arena.key(node).to_string() + } + + fn segment_len(&self, segment: u32) -> usize { + self.segments[segment as usize] + .lists + .iter() + .map(|list| list.len) + .sum() + } + + /// Bound on how many reshuffles one drain loop may perform. Reshuffles move + /// entries between lists without freeing bytes, so a loop that only ever + /// reshuffles needs a stop condition; four passes over the shard is more + /// than enough for every entry to be demoted and then evicted. + fn drain_budget(&self, segment: u32) -> usize { + self.segment_len(segment).saturating_mul(4).saturating_add(4) + } + + fn drop_orphan_node(&mut self, segment: u32, lru: u16, node: u32) { + self.detach(segment, lru, node, 0); + self.release_node(node); + } + + fn evict_segment(&mut self, segment: u32, evicted: &mut Vec) { + let budget = self.drain_budget(segment); let mut attempts = 0usize; - while self.base.used > self.base.capacity && attempts <= self.index.len().saturating_mul(4) - { + while self.segments[segment as usize].used > self.bytes_each_segment && attempts <= budget { attempts = attempts.saturating_add(1); - if let Some(buffer) = self.try_evict_or_shuffle_cold() { + if let Some(buffer) = self.try_evict_or_shuffle_cold(segment) { evicted.push(buffer); continue; } - if self.shuffle_hot_tail() { + if self.shuffle_hot_tail(segment) { continue; } - if self.shuffle_warm_tail() { + if self.shuffle_warm_tail(segment) { continue; } - if let Some(buffer) = self.force_evict_any_tail() { + if let Some(buffer) = self.force_evict_any_tail(segment) { evicted.push(buffer); } else { break; } } - evicted } - fn try_evict_or_shuffle_cold(&mut self) -> Option { - let key = self.cold.pop_back()?; - let entry = self.index.get_mut(&key)?; - if entry.flag >= BUFFER_FETCHED { - entry.flag = BUFFER_ACTIVE; - entry.lru = WARM_LRU; - self.warm.push_front(key); - return None; + fn try_evict_or_shuffle_cold(&mut self, segment: u32) -> Option { + let node = self.list_tail(segment, COLD_LRU)?; + let key = self.node_key(node); + let action = match self.index.get_mut(&key) { + None => SlruTailAction::Orphan, + Some(entry) => { + if entry.flag >= BUFFER_FETCHED { + entry.flag = BUFFER_ACTIVE; + entry.lru = WARM_LRU; + SlruTailAction::MoveTo(WARM_LRU, cache_buffer_space(&entry.buffer)) + } else { + SlruTailAction::Evict + } + } + }; + match action { + SlruTailAction::Orphan => { + self.drop_orphan_node(segment, COLD_LRU, node); + None + } + SlruTailAction::MoveTo(to, space) => { + self.move_node(segment, COLD_LRU, to, node, space); + None + } + SlruTailAction::Evict => self.evict_key(&key), } - self.evict_key(&key) } - fn shuffle_hot_tail(&mut self) -> bool { - let Some(key) = self.hot.pop_back() else { + fn shuffle_hot_tail(&mut self, segment: u32) -> bool { + let Some(node) = self.list_tail(segment, HOT_LRU) else { return false; }; - let Some(entry) = self.index.get_mut(&key) else { - return true; + let key = self.node_key(node); + let action = match self.index.get_mut(&key) { + None => SlruTailAction::Orphan, + Some(entry) => { + let space = cache_buffer_space(&entry.buffer); + if entry.flag >= BUFFER_FETCHED { + entry.flag = BUFFER_ACTIVE; + entry.lru = WARM_LRU; + SlruTailAction::MoveTo(WARM_LRU, space) + } else { + entry.lru = COLD_LRU; + SlruTailAction::MoveTo(COLD_LRU, space) + } + } }; - if entry.flag >= BUFFER_FETCHED { - entry.flag = BUFFER_ACTIVE; - entry.lru = WARM_LRU; - self.warm.push_front(key); - } else { - entry.lru = COLD_LRU; - self.cold.push_front(key); + match action { + SlruTailAction::MoveTo(to, space) => self.move_node(segment, HOT_LRU, to, node, space), + _ => self.drop_orphan_node(segment, HOT_LRU, node), } true } - fn shuffle_warm_tail(&mut self) -> bool { - let Some(key) = self.warm.pop_back() else { + fn shuffle_warm_tail(&mut self, segment: u32) -> bool { + let Some(node) = self.list_tail(segment, WARM_LRU) else { return false; }; - let Some(entry) = self.index.get_mut(&key) else { - return true; + let key = self.node_key(node); + let action = match self.index.get_mut(&key) { + None => SlruTailAction::Orphan, + Some(entry) => { + let space = cache_buffer_space(&entry.buffer); + if entry.flag >= BUFFER_ACTIVE { + entry.flag = BUFFER_FETCHED; + SlruTailAction::MoveTo(WARM_LRU, space) + } else { + entry.lru = COLD_LRU; + SlruTailAction::MoveTo(COLD_LRU, space) + } + } }; - if entry.flag >= BUFFER_ACTIVE { - entry.flag = BUFFER_FETCHED; - self.warm.push_front(key); - } else { - entry.lru = COLD_LRU; - self.cold.push_front(key); + match action { + SlruTailAction::MoveTo(to, space) => self.move_node(segment, WARM_LRU, to, node, space), + _ => self.drop_orphan_node(segment, WARM_LRU, node), } true } - fn force_evict_any_tail(&mut self) -> Option { + fn force_evict_any_tail(&mut self, segment: u32) -> Option { for lru in [COLD_LRU, HOT_LRU, WARM_LRU] { - let key = match lru { - HOT_LRU => self.hot.pop_back(), - WARM_LRU => self.warm.pop_back(), - COLD_LRU => self.cold.pop_back(), - _ => None, + let Some(node) = self.list_tail(segment, lru) else { + continue; }; - if let Some(key) = key { - if let Some(buffer) = self.evict_key(&key) { - return Some(buffer); - } + let key = self.node_key(node); + if let Some(buffer) = self.evict_key(&key) { + return Some(buffer); } + self.drop_orphan_node(segment, lru, node); } None } fn evict_key(&mut self, key: &str) -> Option { let entry = self.index.remove(key)?; - self.base.used = self - .base - .used - .saturating_sub(cache_buffer_space(&entry.buffer)); + let space = cache_buffer_space(&entry.buffer); + self.detach(entry.segment, entry.lru, entry.node, space); + self.release_node(entry.node); if let Some(handler) = &self.mem_eviction_func { handler(clone_cache_buffer(&entry.buffer)); } Some(entry.buffer) } - fn remove_from_lists(&mut self, key: &str) { - self.hot.retain(|candidate| candidate != key); - self.warm.retain(|candidate| candidate != key); - self.cold.retain(|candidate| candidate != key); + fn hot_byte_limit(&self) -> usize { + self.bytes_each_segment + .saturating_mul(self.hot_lru_pct as usize) + / 100 } - fn move_to_front(&mut self, key: &str, lru: u16) { - match lru { - HOT_LRU => { - self.hot.retain(|candidate| candidate != key); - self.hot.push_front(key.to_string()); + fn warm_byte_limit(&self) -> usize { + self.bytes_each_segment + .saturating_mul(self.warm_lru_pct as usize) + / 100 + } + + /// One maintainer pass over a single shard: trim the hot list to its share + /// of the shard budget, then the warm list to its share, then enforce the + /// whole shard budget from the cold list. + fn maintain_segment(&mut self, segment: u32, evicted: &mut Vec) { + let budget = self.drain_budget(segment); + + let hot_limit = self.hot_byte_limit(); + let mut attempts = 0usize; + while self.segments[segment as usize].lists[HOT_LRU as usize].used > hot_limit + && attempts <= budget + { + attempts = attempts.saturating_add(1); + if !self.maintain_demote_tail(segment, HOT_LRU) { + break; } - WARM_LRU => { - self.warm.retain(|candidate| candidate != key); - self.warm.push_front(key.to_string()); + } + + let warm_limit = self.warm_byte_limit(); + attempts = 0; + while self.segments[segment as usize].lists[WARM_LRU as usize].used > warm_limit + && attempts <= budget + { + attempts = attempts.saturating_add(1); + if !self.maintain_demote_tail(segment, WARM_LRU) { + break; } - COLD_LRU => { - self.cold.retain(|candidate| candidate != key); - self.cold.push_front(key.to_string()); + } + + attempts = 0; + while self.segments[segment as usize].used > self.bytes_each_segment && attempts <= budget { + attempts = attempts.saturating_add(1); + match self.maintain_cold_tail(segment) { + SlruMaintainerStep::Stop => break, + SlruMaintainerStep::Moved => {} + SlruMaintainerStep::Evicted(buffer) => evicted.push(buffer), } - _ => {} + } + } + + /// Maintainer step for a hot or warm list tail: an entry touched twice is + /// promoted to the front of the warm list, anything else is demoted to the + /// cold list. Either way the access flag is reset, so an entry that keeps + /// cycling through the warm list without being touched again drains to cold + /// on the next pass. + fn maintain_demote_tail(&mut self, segment: u32, from: u16) -> bool { + let Some(node) = self.list_tail(segment, from) else { + return false; + }; + let key = self.node_key(node); + let action = match self.index.get_mut(&key) { + None => SlruTailAction::Orphan, + Some(entry) => { + let space = cache_buffer_space(&entry.buffer); + let target = if entry.flag == BUFFER_ACTIVE { + WARM_LRU + } else { + COLD_LRU + }; + entry.flag = BUFFER_INIT; + entry.lru = target; + SlruTailAction::MoveTo(target, space) + } + }; + match action { + SlruTailAction::MoveTo(to, space) => self.move_node(segment, from, to, node, space), + _ => self.drop_orphan_node(segment, from, node), + } + true + } + + fn maintain_cold_tail(&mut self, segment: u32) -> SlruMaintainerStep { + let Some(node) = self.list_tail(segment, COLD_LRU) else { + return SlruMaintainerStep::Stop; + }; + let key = self.node_key(node); + let action = match self.index.get_mut(&key) { + None => SlruTailAction::Orphan, + Some(entry) => { + if entry.flag == BUFFER_ACTIVE { + let space = cache_buffer_space(&entry.buffer); + entry.flag = BUFFER_INIT; + entry.lru = WARM_LRU; + SlruTailAction::MoveTo(WARM_LRU, space) + } else { + SlruTailAction::Evict + } + } + }; + match action { + SlruTailAction::Orphan => { + self.drop_orphan_node(segment, COLD_LRU, node); + SlruMaintainerStep::Moved + } + SlruTailAction::MoveTo(to, space) => { + self.move_node(segment, COLD_LRU, to, node, space); + SlruMaintainerStep::Moved + } + SlruTailAction::Evict => match self.evict_key(&key) { + Some(buffer) => SlruMaintainerStep::Evicted(buffer), + None => SlruMaintainerStep::Stop, + }, } } } @@ -1583,6 +2195,26 @@ impl ReplacementSLRU { self.get_item_num() } + pub fn GetSegmentUsedSize(&self, segment_id: usize) -> usize { + self.segment_used_size(segment_id) + } + + pub fn GetSegmentByteLimit(&self) -> usize { + self.segment_byte_limit() + } + + pub fn GetListUsedSize(&self, segment_id: usize, lru: u16) -> usize { + self.list_used_size(segment_id, lru) + } + + pub fn PickSegment(&self, key: &str) -> usize { + self.segment_for_key(key) + } + + pub fn LRUMaintainerTask(&mut self) -> Vec { + self.run_lru_maintainer_pass() + } + pub fn RegisterMemEvictionHandler(&mut self, func: F) where F: Fn(CacheBuffer) + Send + Sync + 'static, @@ -1611,3 +2243,261 @@ impl ReplacementSLRU { } } +/// A segmented LRU that can be shared across threads, holding one lock per +/// segment instead of one lock over the whole policy. +/// +/// [`ReplacementSLRU`] partitions its lists into segments, but it is driven +/// through `&mut self`, so a caller that shares it has to wrap the whole policy +/// in one lock and the partitioning buys nothing under concurrency. This type +/// keeps the same partitioning and gives each segment its own lock, so +/// operations on keys that hash to different segments do not wait on each +/// other. Each segment is an independent [`ReplacementSLRU`] holding +/// `capacity / num_segments` bytes in a single internal segment, which is the +/// same layout, just reached through a per-segment lock. +/// +/// The trade-off is unchanged from the single-threaded form: hash partitioning +/// is slightly less hit-rate-optimal than one global list, because a segment +/// can evict an entry that is globally warmer than one another segment keeps. +pub struct ConcurrentReplacementSLRU { + segments: Vec>, + num_segments: usize, + capacity: usize, + bytes_each_segment: usize, +} + +impl ConcurrentReplacementSLRU { + pub fn new(capacity: usize) -> Self { + Self::with_num_segments(capacity, SLRU_DEFAULT_NUM_SEGMENTS) + } + + /// Build a policy with an explicit segment count. The request is rounded up + /// to a power of two, and collapses to a single segment when `capacity` is + /// smaller than the segment count. + pub fn with_num_segments(capacity: usize, num_segments: usize) -> Self { + let num_segments = resolve_slru_num_segments(capacity, num_segments); + let bytes_each_segment = capacity / num_segments; + let mut segments = Vec::with_capacity(num_segments); + for _ in 0..num_segments { + // Each segment is a whole policy holding one internal segment, so + // its byte budget is exactly this segment share. + segments.push(Mutex::new(ReplacementSLRU::with_num_segments( + bytes_each_segment, + 1, + ))); + } + Self { + segments, + num_segments, + capacity, + bytes_each_segment, + } + } + + pub fn init(&self) -> Result<(), CacheError> { + for segment in &self.segments { + self.lock(segment).init()?; + } + Ok(()) + } + + pub fn reset(&self) -> Result<(), CacheError> { + for segment in &self.segments { + self.lock(segment).reset()?; + } + Ok(()) + } + + pub fn put(&self, buffer: CacheBuffer) -> Vec { + let index = self.segment_for_key(buffer.key()); + self.lock(&self.segments[index]).put(buffer) + } + + pub fn get(&self, key: &str) -> Option { + let index = self.segment_for_key(key); + self.lock(&self.segments[index]).get(key) + } + + pub fn peek(&self, key: &str) -> Option { + let index = self.segment_for_key(key); + self.lock(&self.segments[index]).peek(key) + } + + pub fn delete(&self, key: &str) -> Option { + let index = self.segment_for_key(key); + self.lock(&self.segments[index]).delete(key) + } + + pub fn update_cache_buffer( + &self, + key: &str, + old_data: &[u8], + buffer: CacheBuffer, + ) -> Result<(), CacheError> { + let index = self.segment_for_key(key); + self.lock(&self.segments[index]).update_cache_buffer(key, old_data, buffer) + } + + /// Run one maintainer sweep over every segment, taking each segment lock in + /// turn rather than holding one lock across the whole sweep. + pub fn run_lru_maintainer_pass(&self) -> Vec { + let mut evicted = Vec::new(); + for segment in &self.segments { + evicted.extend(self.lock(segment).run_lru_maintainer_pass()); + } + evicted + } + + pub fn get_capacity(&self) -> usize { + self.capacity + } + + pub fn get_used_space(&self) -> usize { + self.segments + .iter() + .map(|segment| self.lock(segment).get_used_space()) + .sum() + } + + pub fn get_free_space(&self) -> usize { + self.capacity.saturating_sub(self.get_used_space()) + } + + pub fn get_item_num(&self) -> usize { + self.segments + .iter() + .map(|segment| self.lock(segment).get_item_num()) + .sum() + } + + pub fn num_segments(&self) -> usize { + self.num_segments + } + + pub fn segment_byte_limit(&self) -> usize { + self.bytes_each_segment + } + + pub fn segment_used_size(&self, segment_id: usize) -> usize { + self.segments + .get(segment_id) + .map(|segment| self.lock(segment).get_used_space()) + .unwrap_or(0) + } + + pub fn segment_item_num(&self, segment_id: usize) -> usize { + self.segments + .get(segment_id) + .map(|segment| self.lock(segment).get_item_num()) + .unwrap_or(0) + } + + /// Segment that owns `key`. Uses the same hash and mask as the + /// single-threaded form, so a key lands in the same relative segment. + pub fn segment_for_key(&self, key: &str) -> usize { + if self.num_segments <= 1 { + return 0; + } + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + (hasher.finish() & (self.num_segments as u64 - 1)) as usize + } + + pub fn set_hot_lru_pct(&self, pct: u32) { + for segment in &self.segments { + self.lock(segment).set_hot_lru_pct(pct); + } + } + + pub fn set_warm_lru_pct(&self, pct: u32) { + for segment in &self.segments { + self.lock(segment).set_warm_lru_pct(pct); + } + } + + pub fn test_config_lru_maintainer(&self, status: bool) { + for segment in &self.segments { + self.lock(segment).test_config_lru_maintainer(status); + } + } + + /// Register an eviction handler on every segment. The handler is shared, so + /// it is called from whichever thread drives the eviction and must be + /// `Send + Sync`. + pub fn register_mem_eviction_handler(&self, func: F) + where + F: Fn(CacheBuffer) + Send + Sync + 'static, + { + let shared = Arc::new(func); + for segment in &self.segments { + let handler = Arc::clone(&shared); + self.lock(segment).register_mem_eviction_handler(move |buffer| handler(buffer)); + } + } + + /// A poisoned segment lock means another thread panicked mid-update, which + /// would leave that segment inconsistent. Recovering the guard keeps the + /// remaining segments usable rather than cascading the panic across every + /// caller. + fn lock<'a>(&self, segment: &'a Mutex) -> std::sync::MutexGuard<'a, ReplacementSLRU> { + segment.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +#[allow(non_snake_case)] +impl ConcurrentReplacementSLRU { + pub fn Init(&self) -> Result<(), CacheError> { + self.init() + } + + pub fn Reset(&self) -> Result<(), CacheError> { + self.reset() + } + + pub fn Put(&self, buffer: CacheBuffer) -> Vec { + self.put(buffer) + } + + pub fn Get(&self, key: &str) -> Option { + self.get(key) + } + + pub fn Peek(&self, key: &str) -> Option { + self.peek(key) + } + + pub fn Delete(&self, key: &str) -> Option { + self.delete(key) + } + + pub fn GetCapacity(&self) -> usize { + self.get_capacity() + } + + pub fn GetUsedSpace(&self) -> usize { + self.get_used_space() + } + + pub fn GetFreeSpace(&self) -> usize { + self.get_free_space() + } + + pub fn GetItemNum(&self) -> usize { + self.get_item_num() + } + + pub fn GetSegmentUsedSize(&self, segment_id: usize) -> usize { + self.segment_used_size(segment_id) + } + + pub fn GetSegmentByteLimit(&self) -> usize { + self.segment_byte_limit() + } + + pub fn PickSegment(&self, key: &str) -> usize { + self.segment_for_key(key) + } + + pub fn LRUMaintainerTask(&self) -> Vec { + self.run_lru_maintainer_pass() + } +} diff --git a/src/runtime/storage_engines.rs b/src/runtime/storage_engines.rs index 120977d..3b8a627 100644 --- a/src/runtime/storage_engines.rs +++ b/src/runtime/storage_engines.rs @@ -1062,19 +1062,45 @@ impl MemStorageRecordHandle { pub struct MemStorage; +/// Continue a CRC-32C (Castagnoli) checksum over `bytes`, starting from the +/// checksum `seed` returned by an earlier call. Chaining a seed gives the same +/// result as checksumming the concatenated input in one go, which is what lets +/// a record be checksummed header-then-value-then-key without copying it into +/// one buffer first. +pub fn crc32c_with_seed(bytes: &[u8], seed: u32) -> u32 { + let mut crc = !seed; + for byte in bytes { + crc ^= *byte as u32; + for _ in 0..8 { + let mask = 0u32.wrapping_sub(crc & 1); + crc = (crc >> 1) ^ (0x82f6_3b78 & mask); + } + } + !crc +} + +/// CRC-32C (Castagnoli) checksum of `bytes`. +pub fn crc32c(bytes: &[u8]) -> u32 { + crc32c_with_seed(bytes, 0) +} + impl MemStorage { pub const HEADER_BYTES: usize = 8; + /// Checksum of a cache record: the length header first, then the value, + /// then the key. + /// + /// Covering the header matters. Two records whose value and key bytes + /// concatenate to the same sequence but split differently are only + /// distinguishable by their lengths, so a checksum over the payload alone + /// would accept a record whose length header had been corrupted. pub fn compute_crc(key: &str, value: &[u8]) -> u32 { - let mut crc = 0xffff_ffffu32; - for byte in value.iter().chain(key.as_bytes().iter()).copied() { - crc ^= byte as u32; - for _ in 0..8 { - let mask = 0u32.wrapping_sub(crc & 1); - crc = (crc >> 1) ^ (0xedb8_8320 & mask); - } - } - !crc + let mut lengths = [0u8; Self::HEADER_BYTES]; + lengths[..4].copy_from_slice(&(value.len() as u32).to_le_bytes()); + lengths[4..].copy_from_slice(&(key.len() as u32).to_le_bytes()); + let crc = crc32c(&lengths); + let crc = crc32c_with_seed(value, crc); + crc32c_with_seed(key.as_bytes(), crc) } pub fn do_put(key: &str, value: &[u8]) -> Vec { @@ -2499,13 +2525,24 @@ impl StorageEngineMultiSSD { storage } + /// Hash used to spread keys across devices. + /// + /// This is the same hash the rest of the crate uses, so the device a key + /// lands on is reproducible: a data directory written by one process is + /// read back through the same device selection by another. fn hash(key: &str) -> u32 { - let mut hash = 0x811c_9dc5u32; - for byte in key.as_bytes() { - hash ^= *byte as u32; - hash = hash.wrapping_mul(0x0100_0193); + mur_mur_hash2(key.as_bytes()) + } + + /// How many devices keys are spread across. Before start there are no + /// storages yet, so the configured paths stand in; `init` rebuilds the + /// storages from the paths one for one, so the two agree once started. + fn shard_count(&self) -> usize { + if self.initialized && !self.storages.is_empty() { + self.storages.len() + } else { + self.paths.len() } - hash } fn storage_index(&self, key: &str) -> Result { @@ -2550,11 +2587,14 @@ impl StorageEngineMultiSSD { self.storages.len() } + /// Path of the device that holds `key`, using the same selection as reads + /// and writes so the answer matches where the data actually goes. pub fn device_for_key(&self, key: &str) -> Option<&str> { - if self.paths.is_empty() { + let count = self.shard_count(); + if count == 0 { return None; } - let index = Self::hash(key) as usize % self.paths.len(); + let index = Self::hash(key) as usize % count; self.paths.get(index).map(String::as_str) } diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 0ef708b..638284e 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -3803,9 +3803,14 @@ mod tests { let mut policy = L2CachePolicy::new(l1, l2, arc, 8, 8, 8); policy.Start(); + // Access records are buffered by default; a drain pass applies them. policy.OnAccess(AccessRecordType::Put, "cold-a"); policy.OnAccess(AccessRecordType::Put, "cold-b"); + assert_eq!(policy.access_callback_count(), 0); + assert_eq!(policy.access_buffer_size(), 2); + policy.access_task_internal(); assert_eq!(policy.access_callback_count(), 2); + assert_eq!(policy.access_buffer_size(), 0); assert_eq!( policy.arc_policy().GetFetchTail(8), vec!["cold-a".to_string(), "cold-b".to_string()] @@ -3826,6 +3831,7 @@ mod tests { ); policy.OnAccess(AccessRecordType::Delete, "cold-a"); + policy.access_task_internal(); assert!(!policy .arc_policy() .GetFetchTail(8) @@ -3861,6 +3867,8 @@ mod tests { .unwrap() .push(buffer.Key().to_string()); }); + // Queueing evicted buffers for the lower tier is opt-in. + policy.set_use_eviction_handler(true); policy.Start(); let mut first = CacheBuffer::new(b"first".to_vec()); @@ -3899,6 +3907,146 @@ mod tests { assert_eq!(policy.write_task_internal().unwrap(), 1); } + fn l2_test_policy( + l2_dir: &Path, + arc_items: usize, + access_capacity: usize, + tail_batch: usize, + write_capacity: usize, + ) -> L2CachePolicy { + let dram = CacheInstance::new( + 128, + ReplacementPolicyType::kFIFO, + StorageEngineType::kDRAM, + vec![], + ); + let l2 = CacheInstance::new( + 256, + ReplacementPolicyType::kFIFO, + StorageEngineType::kSSD, + vec![l2_dir.to_path_buf()], + ); + let l1 = L1CacheImplement::new(dram, None); + let mut arc = ReplacementArc::new(arc_items); + arc.Init().unwrap(); + L2CachePolicy::new(l1, l2, arc, access_capacity, tail_batch, write_capacity) + } + + #[test] + fn parity_l2_cache_policy_access_buffering_modes_and_drop_on_full() { + let dir = tempfile::tempdir().unwrap(); + let mut policy = l2_test_policy(dir.path(), 8, 2, 8, 8); + policy.Start(); + assert!(policy.async_on_access()); + + // Buffered by default, and the buffer is bounded: the third record is + // dropped rather than letting the buffer grow without limit. + policy.OnAccess(AccessRecordType::Put, "a"); + policy.OnAccess(AccessRecordType::Put, "b"); + policy.OnAccess(AccessRecordType::Put, "c"); + assert_eq!(policy.access_buffer_size(), 2); + assert_eq!(policy.access_drop_count(), 1); + assert_eq!(policy.access_callback_count(), 0); + + policy.access_task_internal(); + assert_eq!(policy.access_callback_count(), 2); + assert_eq!(policy.access_buffer_size(), 0); + + // Inline mode applies the record on the calling path instead, so the + // buffer stays empty and nothing can be dropped. + policy.set_async_on_access(false); + policy.OnAccess(AccessRecordType::Put, "d"); + assert_eq!(policy.access_buffer_size(), 0); + assert_eq!(policy.access_callback_count(), 3); + assert_eq!(policy.access_drop_count(), 1); + } + + #[test] + fn parity_l2_cache_policy_eviction_handler_is_off_by_default() { + let dir = tempfile::tempdir().unwrap(); + let mut policy = l2_test_policy(dir.path(), 8, 8, 8, 8); + policy.Start(); + assert!(!policy.use_eviction_handler()); + + let mut dropped = CacheBuffer::new(b"dropped".to_vec()); + dropped.SetKey("dropped-key"); + policy.OnEvict(dropped); + // The default drops evicted data rather than writing it down a tier, + // leaving the tail passes as the only path into the lower tier. + assert_eq!(policy.write_buffer_size(), 0); + assert_eq!(policy.write_enqueue_fail_count(), 0); + + policy.set_use_eviction_handler(true); + let mut kept = CacheBuffer::new(b"kept".to_vec()); + kept.SetKey("kept-key"); + policy.OnEvict(kept); + assert_eq!(policy.write_buffer_size(), 1); + } + + #[test] + fn parity_l2_cache_policy_poll_paces_passes_by_interval() { + let dir = tempfile::tempdir().unwrap(); + let mut policy = l2_test_policy(dir.path(), 8, 8, 8, 8); + policy.Start(); + assert_eq!(policy.access_interval_ms(), L2_DEFAULT_ACCESS_INTERVAL_MS); + assert_eq!(policy.tail_interval_ms(), L2_DEFAULT_TAIL_INTERVAL_MS); + assert_eq!(policy.write_interval_ms(), L2_DEFAULT_WRITE_INTERVAL_MS); + + // With long intervals the first poll runs every pass, and the second + // runs none, because none are due yet. + policy.set_access_interval_ms(60_000); + policy.set_tail_interval_ms(60_000); + policy.set_write_interval_ms(60_000); + policy.OnAccess(AccessRecordType::Put, "paced"); + assert_eq!(policy.poll().unwrap(), 0); + assert_eq!(policy.access_callback_count(), 1); + + policy.OnAccess(AccessRecordType::Put, "paced-again"); + assert_eq!(policy.poll().unwrap(), 0); + assert_eq!(policy.access_callback_count(), 1); + assert_eq!(policy.access_buffer_size(), 1); + + // Dropping the interval to zero makes the pass due again. + policy.set_access_interval_ms(0); + assert_eq!(policy.poll().unwrap(), 0); + assert_eq!(policy.access_callback_count(), 2); + + // flush_once ignores pacing entirely. + policy.set_access_interval_ms(60_000); + policy.OnAccess(AccessRecordType::Put, "flushed"); + policy.flush_once().unwrap(); + assert_eq!(policy.access_callback_count(), 3); + } + + #[test] + fn parity_l2_cache_policy_factory_uses_reference_default_sizing() { + let dram = CacheInstance::new( + 64, + ReplacementPolicyType::kFIFO, + StorageEngineType::kDRAM, + vec![], + ); + let l2_dir = tempfile::tempdir().unwrap(); + let l2 = CacheInstance::new( + 128, + ReplacementPolicyType::kFIFO, + StorageEngineType::kSSD, + vec![l2_dir.path().to_path_buf()], + ); + let l1 = L1CacheImplement::new(dram, None); + let policy = L2CachePolicyFactory::CreateL2CachePolicy(l1, l2); + + assert_eq!( + policy.arc_policy().GetItemCapacity(), + L2_DEFAULT_MAX_ARC_CACHE_ITEMS + ); + assert_eq!(policy.access_interval_ms(), L2_DEFAULT_ACCESS_INTERVAL_MS); + assert_eq!(policy.tail_interval_ms(), L2_DEFAULT_TAIL_INTERVAL_MS); + assert_eq!(policy.write_interval_ms(), L2_DEFAULT_WRITE_INTERVAL_MS); + assert!(policy.async_on_access()); + assert!(!policy.use_eviction_handler()); + } + #[test] fn parity_l2_cache_policy_factory_builds_started_policy_surface() { let dram = CacheInstance::new( @@ -4050,21 +4198,25 @@ mod tests { Vec::new(), ); let evictions = Arc::new(std::sync::Mutex::new(Vec::new())); - let metric_sizes = Arc::new(std::sync::Mutex::new(Vec::new())); + let metric_counts = Arc::new(std::sync::Mutex::new(Vec::new())); let captured_evictions = Arc::clone(&evictions); - let captured_metrics = Arc::clone(&metric_sizes); + let captured_metrics = Arc::clone(&metric_counts); instance.RegisterEvictionHandler(move |record| { captured_evictions.lock().unwrap().push(record.key); }); - instance.RegisterEvictionMetricHandler(move |size| { - captured_metrics.lock().unwrap().push(size); + instance.RegisterEvictionMetricHandler(move |count| { + captured_metrics.lock().unwrap().push(count); }); + // With the handler switched off nothing reaches it, but eviction + // metrics are independent of the handler and keep counting: the + // eviction rate stays observable even when nothing is consuming the + // evicted entries. instance.SetEvictionHandlerStatus(false); instance.Put("first", b"12345678".to_vec()).unwrap(); instance.Put("second", b"abcdefgh".to_vec()).unwrap(); assert!(evictions.lock().unwrap().is_empty()); - assert!(metric_sizes.lock().unwrap().is_empty()); + assert_eq!(metric_counts.lock().unwrap().as_slice(), &[1]); instance.SetEvictionHandlerStatus(true); instance.Put("third", b"ABCDEFGH".to_vec()).unwrap(); @@ -4072,7 +4224,9 @@ mod tests { evictions.lock().unwrap().as_slice(), &[CacheKey::string(0, "second")] ); - assert_eq!(metric_sizes.lock().unwrap().as_slice(), &[8]); + // The metric counts evicted entries, not their bytes, which is what + // lets it be reported without materialising anything. + assert_eq!(metric_counts.lock().unwrap().as_slice(), &[1, 1]); } #[test] @@ -6263,6 +6417,91 @@ mod tests { assert!(arc.GetActiveDataTail(8).contains(&"a".to_string())); } + #[test] + fn parity_arc_list_hit_on_fetch_data_promotes_to_active() { + let mut arc = ArcList::new(4); + arc.Put("a".to_string()); + assert_eq!(arc.GetFetchDataTail(8), vec!["a".to_string()]); + assert!(arc.GetActiveDataTail(8).is_empty()); + + // A hit on a key held in the fetch data list promotes it to the active + // data list: one access means fetched, two means worth keeping. + assert!(arc.Get("a")); + assert!(arc.GetFetchDataTail(8).is_empty()); + assert_eq!(arc.GetActiveDataTail(8), vec!["a".to_string()]); + } + + #[test] + fn parity_arc_list_ghost_hits_adapt_the_fetch_active_split() { + // Capacity 2 starts split evenly, one slot each side. + let mut arc = ArcList::new(2); + assert_eq!(arc.FetchCapacity(), 1); + assert_eq!(arc.ActiveCapacity(), 1); + + arc.Put("a".to_string()); + assert!(arc.Get("a")); + arc.Put("b".to_string()); + // This insert takes the list to capacity, so making room downgrades the + // active tail into the active ghost list rather than dropping it. + arc.Put("c".to_string()); + assert!(arc.GetActiveGhostTail(8).contains(&"a".to_string())); + + // Hitting a key in the active ghost list is evidence the active side + // was trimmed too far, so it takes a slot from the fetch side. + assert!(!arc.Get("a")); + assert_eq!(arc.FetchCapacity(), 0); + assert_eq!(arc.ActiveCapacity(), 2); + assert!(arc.GetActiveDataTail(8).contains(&"a".to_string())); + // Making room for it downgraded the fetch tail into the fetch ghost. + assert!(arc.GetFetchGhostTail(8).contains(&"b".to_string())); + + // Hitting the fetch ghost is the mirror image, and hands the slot back. + assert!(!arc.Get("b")); + assert_eq!(arc.FetchCapacity(), 1); + assert_eq!(arc.ActiveCapacity(), 1); + assert!(arc.GetActiveDataTail(8).contains(&"b".to_string())); + } + + #[test] + fn parity_arc_list_drops_fetch_tail_outright_when_its_ghost_is_empty() { + let mut arc = ArcList::new(2); + arc.Put("a".to_string()); + arc.Put("b".to_string()); + assert_eq!(arc.Size(), 2); + assert_eq!(arc.GhostSize(), 0); + + // The fetch side alone already holds the whole capacity and its ghost + // list is empty, so its tail is dropped outright instead of ghosted. + arc.Put("c".to_string()); + assert_eq!(arc.Size(), 2); + assert_eq!(arc.GhostSize(), 0); + assert!(!arc.GetFetchDataTail(8).contains(&"a".to_string())); + assert!(arc.GetFetchDataTail(8).contains(&"b".to_string())); + assert!(arc.GetFetchDataTail(8).contains(&"c".to_string())); + assert!(arc.Size() <= arc.Capacity()); + assert!(arc.TotalSize() <= arc.Capacity() * 2); + } + + #[test] + fn ghost_lru_list_delete_clears_the_key_from_whichever_list_holds_it() { + let mut list = GhostLRUList::new(4); + list.Put("data-key".to_string()); + list.PutGhost("ghost-key".to_string()); + assert_eq!(list.Size(), 1); + assert_eq!(list.GhostSize(), 1); + + // Delete reports whether the key was there and removes it from + // whichever list held it, including a key that only exists as a ghost. + assert!(list.Delete("data-key")); + assert_eq!(list.Size(), 0); + assert_eq!(list.GhostSize(), 1); + + assert!(list.Delete("ghost-key")); + assert_eq!(list.GhostSize(), 0); + + assert!(!list.Delete("absent")); + } + #[test] fn parity_replacement_arc_exposes_active_and_fetch_tail_surface() { let mut policy = ReplacementArc::new(2); @@ -6369,6 +6608,40 @@ mod tests { assert_eq!(buffer.Count(), 0); } + #[test] + fn parity_mem_storage_crc_is_castagnoli_and_covers_the_length_header() { + // Pin the algorithm to its published check value rather than to + // whatever this implementation happens to emit. + assert_eq!(crc32c(b"123456789"), 0xe306_9283); + assert_eq!(crc32c(b""), 0); + + // Seeding continues a checksum, so a record can be covered in three + // pieces without first copying them into one buffer. + assert_eq!( + crc32c_with_seed(b"56789", crc32c(b"1234")), + crc32c(b"123456789") + ); + + // "cdef" + "ab" and "cdefa" + "b" concatenate to the same bytes and + // differ only in where the value ends and the key begins. A checksum + // over the payload alone cannot tell them apart, so covering the + // length header is what detects a corrupted header. + assert_ne!( + MemStorage::ComputeCRC("ab", b"cdef"), + MemStorage::ComputeCRC("b", b"cdefa") + ); + + // Value and key both still contribute. + assert_ne!( + MemStorage::ComputeCRC("k", b"v1"), + MemStorage::ComputeCRC("k", b"v2") + ); + assert_ne!( + MemStorage::ComputeCRC("k1", b"v"), + MemStorage::ComputeCRC("k2", b"v") + ); + } + #[test] fn parity_mem_storage_layout_round_trips_key_value_and_crc() { let crc = MemStorage::ComputeCRC("layout-key", b"layout-value"); @@ -6794,6 +7067,72 @@ mod tests { assert_eq!(stats.total_bytes, b"pmem-value".len()); } + #[test] + fn parity_ssd_fifo_keeps_insertion_order_when_a_key_is_rewritten() { + let dir = tempfile::tempdir().unwrap(); + let instance = CacheInstance::new( + 520, + ReplacementPolicyType::kFIFO, + StorageEngineType::kSSD, + vec![dir.path().to_path_buf()], + ); + instance.Start().unwrap(); + let big = vec![120u8; 100]; + let small = vec![121u8; 10]; + for key in ["a", "b", "c"] { + instance.Put(key, big.clone()).unwrap(); + } + + // Rewriting "a" must not move it behind "b" and "c". First-in + // first-out orders by when a key first entered the tier, not by when + // it was last written, so "a" stays the next one out. + instance.Put("a", small.clone()).unwrap(); + instance.Put("d", big.clone()).unwrap(); + instance.Put("e", big.clone()).unwrap(); + + assert!( + instance.Get("a").unwrap().is_none(), + "the first key inserted should still be the first evicted" + ); + assert!( + instance.Get("b").unwrap().is_some(), + "rewriting another key must not push this one to the front of the queue" + ); + assert!(instance.Get("e").unwrap().is_some()); + } + + #[test] + fn parity_multi_ssd_selects_the_device_with_the_shared_hash() { + let dir = tempfile::tempdir().unwrap(); + let dev = |name: &str| dir.path().join(name).to_string_lossy().to_string(); + let paths = vec![dev("ssd-a"), dev("ssd-b"), dev("ssd-c")]; + let engine = StorageEngineMultiSSD::new(paths.clone(), 1 << 20); + + // Device selection uses the same hash as the rest of the crate rather + // than an ad-hoc one, so which device holds a key is reproducible: a + // set of device directories written by one process is read back + // through the same selection by another. + for key in ["alpha", "beta", "gamma", "delta", "hello", "abc"] { + let expected = &paths[mur_mur_hash2(key.as_bytes()) as usize % paths.len()]; + assert_eq!( + engine.device_for_key(key), + Some(expected.as_str()), + "device for {key}" + ); + } + + // Three devices is not a power of two, so selection has to be a + // modulo rather than a mask or the third device never gets a key. + let mut seen = HashSet::new(); + for index in 0..256 { + let key = format!("spread-{index:04}"); + if let Some(device) = engine.device_for_key(&key) { + seen.insert(device.to_string()); + } + } + assert_eq!(seen.len(), 3, "every device should receive keys"); + } + #[test] fn parity_multi_ssd_requires_devices_and_hashes_keys_to_storage() { let mut empty = StorageEngineMultiSSD::new(Vec::::new(), 1024); @@ -7710,6 +8049,50 @@ mod tests { assert!(!controller.enable_gc()); } + #[test] + fn parity_storage_gc_controller_poll_paces_collection_checks() { + let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); + let first = allocator.Allocate(8).unwrap(); + allocator.Free(first, 8).unwrap(); + + let mut controller = StorageGCController::new(allocator, true); + assert_eq!( + controller.gc_check_interval_ms(), + GC_DEFAULT_CHECK_INTERVAL_MS + ); + + // Collection is disabled until started, so polling does nothing. + assert_eq!(controller.poll().unwrap(), 0); + + controller.start(); + controller.set_gc_check_interval_ms(60_000); + // The first check is always due. + assert_eq!(controller.poll().unwrap(), 1); + + // Queue more work. The interval has not elapsed, so the controller + // leaves it for the next due check instead of scanning on every call. + let second = controller.allocator_mut().Allocate(8).unwrap(); + controller.allocator_mut().Free(second, 8).unwrap(); + assert_eq!(controller.poll().unwrap(), 0); + + // Shortening the interval makes the check due and the work is taken. + controller.set_gc_check_interval_ms(0); + assert_eq!(controller.poll().unwrap(), 1); + + // Pausing and disabling each suppress collection regardless of pacing. + let third = controller.allocator_mut().Allocate(8).unwrap(); + controller.allocator_mut().Free(third, 8).unwrap(); + controller.set_pause_gc(true); + assert_eq!(controller.poll().unwrap(), 0); + controller.set_pause_gc(false); + controller.set_enable_gc(false); + assert_eq!(controller.poll().unwrap(), 0); + + controller.set_enable_gc(true); + assert_eq!(controller.poll().unwrap(), 1); + assert_eq!(controller.TEST_GetNumGcCompleteChunks(), 3); + } + #[test] fn parity_storage_gc_controller_respects_enable_gate_and_manual_gc_job() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); @@ -8029,6 +8412,125 @@ mod tests { assert!(fifo.Get("next").is_some()); } + #[test] + fn parity_replacement_fifo_overwrite_keeps_original_queue_position() { + let mut fifo = ReplacementFIFO::new(6); + fifo.Init().unwrap(); + // Three 2-byte entries exactly fill the policy. + for key in ["a", "b", "c"] { + assert!(fifo.Put(test_buffer(key, b"1")).is_empty()); + } + assert_eq!(fifo.GetItemNum(), 3); + assert_eq!(fifo.GetUsedSpace(), 6); + + // Rewriting "a" must not move it behind "b" and "c": first-in + // first-out orders by first insertion, not by last write. + assert!(fifo.Put(test_buffer("a", b"9")).is_empty()); + assert_eq!(fifo.Get("a").unwrap().Data(), b"9"); + assert_eq!(fifo.GetItemNum(), 3); + assert_eq!(fifo.GetUsedSpace(), 6); + + // So the next insert still evicts "a", the oldest by insertion order. + let evicted = fifo.Put(test_buffer("d", b"4")); + assert_eq!(evicted.len(), 1); + assert_eq!(evicted[0].Key(), "a"); + assert!(fifo.Peek("b").is_some()); + assert!(fifo.Peek("c").is_some()); + assert!(fifo.Peek("d").is_some()); + } + + #[test] + fn parity_replacement_fifo_delete_leaves_no_queue_tombstone() { + let mut fifo = ReplacementFIFO::new(1 << 12); + fifo.Init().unwrap(); + for index in 0..256 { + fifo.Put(test_buffer(&format!("k{index:03}"), b"v")); + } + assert_eq!(fifo.GetItemNum(), 256); + assert_eq!(fifo.queue_len(), 256); + + for index in (0..256).step_by(2) { + assert!(fifo.Delete(&format!("k{index:03}")).is_some()); + } + // The queue tracks live entries only, so a delete-heavy workload leaves + // no stale keys for eviction to skip past. + assert_eq!(fifo.GetItemNum(), 128); + assert_eq!(fifo.queue_len(), 128); + + for index in (0..256).step_by(2) { + fifo.Put(test_buffer(&format!("k{index:03}"), b"v")); + } + assert_eq!(fifo.GetItemNum(), 256); + assert_eq!(fifo.queue_len(), 256); + assert!(fifo.GetUsedSpace() <= fifo.GetCapacity()); + } + + #[test] + fn parity_replacement_slru_get_records_access_without_reordering() { + let mut slru = ReplacementSLRU::with_num_segments(100, 1); + slru.Init().unwrap(); + slru.TEST_ConfigLRUMaintainer(false); + + // Each entry is 1 key byte plus 4 value bytes; hot runs c, b, a from + // head to tail. + for key in ["a", "b", "c"] { + slru.Put(test_buffer(key, b"xxxx")); + } + assert_eq!(slru.list_item_num(0, HOT_LRU), 3); + + // Reading the hot tail marks it but leaves it exactly where it was. + assert!(slru.Get("a").is_some()); + assert_eq!(slru.TEST_CheckBufferFlag("a"), BUFFER_FETCHED); + assert!(slru.Get("a").is_some()); + assert_eq!(slru.TEST_CheckBufferFlag("a"), BUFFER_ACTIVE); + assert_eq!(slru.TEST_CheckLRUPos("a"), HOT_LRU); + assert_eq!(slru.list_item_num(0, HOT_LRU), 3); + assert_eq!(slru.list_item_num(0, WARM_LRU), 0); + + // "a" is therefore still the hot tail when the maintainer runs, and the + // maintainer is what promotes it -- because the two reads marked it + // active. Untouched "b" behind it is demoted instead. + slru.set_hot_lru_pct(5); + slru.TEST_ConfigLRUMaintainer(true); + assert!(slru.run_lru_maintainer_pass().is_empty()); + assert_eq!(slru.TEST_CheckLRUPos("a"), WARM_LRU); + assert_eq!(slru.TEST_CheckLRUPos("b"), COLD_LRU); + assert_eq!(slru.TEST_CheckLRUPos("c"), HOT_LRU); + assert_eq!(slru.GetItemNum(), 3); + } + + #[test] + fn base_lru_list_recycles_nodes_across_repeated_churn() { + let mut list = BaseLRUList::new(64); + for round in 0..8 { + for index in 0..64 { + list.Put(format!("r{round}-k{index:02}")); + } + assert_eq!(list.Size(), 64); + assert!(list.Evict().is_empty()); + for index in 0..64 { + assert!(list.Delete(&format!("r{round}-k{index:02}"))); + } + assert_eq!(list.Size(), 0); + assert!(list.GetTail(8).is_empty()); + } + + // Nodes freed by the churn above are reused, and the list is still + // correctly ordered afterwards. + list.Put("first".to_string()); + list.Put("second".to_string()); + assert_eq!( + list.GetTail(2), + vec!["first".to_string(), "second".to_string()] + ); + assert!(list.Get("first")); + assert_eq!( + list.GetTail(2), + vec!["second".to_string(), "first".to_string()] + ); + assert_eq!(list.Size(), 2); + } + #[test] fn parity_replacement_slru_tracks_hot_warm_cold_and_fetch_flags() { let mut slru = ReplacementSLRU::new(6); @@ -8074,6 +8576,525 @@ mod tests { assert!(slru.GetUsedSpace() <= slru.GetCapacity()); } + #[test] + fn parity_replacement_slru_resolves_segment_count_from_capacity_and_request() { + // A capacity smaller than the segment count collapses to one segment, + // otherwise every segment would get a zero byte budget. + assert_eq!(ReplacementSLRU::new(6).num_segments(), 1); + assert_eq!(ReplacementSLRU::new(255).num_segments(), 1); + assert_eq!( + ReplacementSLRU::new(256).num_segments(), + SLRU_DEFAULT_NUM_SEGMENTS + ); + + // Requests are rounded up to a power of two so segment selection masks. + assert_eq!( + ReplacementSLRU::with_num_segments(1 << 20, 100).num_segments(), + 128 + ); + assert_eq!(ReplacementSLRU::with_num_segments(1 << 20, 0).num_segments(), 1); + assert_eq!(ReplacementSLRU::with_num_segments(1 << 20, 1).num_segments(), 1); + + let policy = ReplacementSLRU::with_num_segments(1024, 8); + assert_eq!(policy.segment_byte_limit(), 128); + assert_eq!(policy.GetSegmentByteLimit(), 128); + assert_eq!(policy.hot_lru_pct(), SLRU_DEFAULT_HOT_LRU_PCT); + assert_eq!(policy.warm_lru_pct(), SLRU_DEFAULT_WARM_LRU_PCT); + } + + #[test] + fn parity_replacement_slru_shards_keys_and_bounds_each_segment() { + let mut slru = ReplacementSLRU::new(1 << 16); + slru.Init().unwrap(); + assert_eq!(slru.num_segments(), SLRU_DEFAULT_NUM_SEGMENTS); + assert_eq!( + slru.segment_byte_limit(), + (1 << 16) / SLRU_DEFAULT_NUM_SEGMENTS + ); + + for index in 0..4096 { + slru.Put(test_buffer(&format!("shard-key-{index:06}"), &[b'v'; 32])); + } + + // Eviction is segment-local: every shard is held to its own budget + // rather than to one global list. + for segment in 0..slru.num_segments() { + assert!( + slru.segment_used_size(segment) <= slru.segment_byte_limit(), + "segment {segment} over budget" + ); + } + + let summed: usize = (0..slru.num_segments()) + .map(|segment| slru.segment_used_size(segment)) + .sum(); + assert_eq!(slru.GetUsedSpace(), summed); + assert!(slru.GetUsedSpace() <= slru.GetCapacity()); + + let occupied = (0..slru.num_segments()) + .filter(|&segment| slru.segment_used_size(segment) > 0) + .count(); + assert!( + occupied > 200, + "expected keys spread across shards, only {occupied} occupied" + ); + + slru.Put(test_buffer("probe-key", b"probe")); + let shard = slru.segment_for_key("probe-key"); + assert_eq!(shard, slru.PickSegment("probe-key")); + assert!(shard < slru.num_segments()); + assert!(slru.segment_used_size(shard) >= "probe-key".len() + b"probe".len()); + } + + #[test] + fn parity_replacement_slru_accounting_survives_overwrite_delete_and_reuse() { + let mut slru = ReplacementSLRU::with_num_segments(1 << 14, 4); + slru.Init().unwrap(); + + for index in 0..512 { + slru.Put(test_buffer(&format!("k{index:04}"), &[b'x'; 16])); + } + for index in 0..512 { + let _ = slru.Get(&format!("k{index:04}")); + } + for index in (0..512).step_by(2) { + let _ = slru.Delete(&format!("k{index:04}")); + } + for index in 0..512 { + slru.Put(test_buffer(&format!("k{index:04}"), &[b'y'; 24])); + } + + // Per-list byte and item counts still reconcile with the shard totals + // and the index, so no list node was leaked or double-counted. + let mut items = 0usize; + for segment in 0..slru.num_segments() { + let listed: usize = [HOT_LRU, WARM_LRU, COLD_LRU] + .iter() + .map(|&lru| slru.list_used_size(segment, lru)) + .sum(); + assert_eq!(listed, slru.segment_used_size(segment)); + assert_eq!( + listed, + slru.GetListUsedSize(segment, HOT_LRU) + + slru.GetListUsedSize(segment, WARM_LRU) + + slru.GetListUsedSize(segment, COLD_LRU) + ); + assert!(slru.segment_used_size(segment) <= slru.segment_byte_limit()); + items += [HOT_LRU, WARM_LRU, COLD_LRU] + .iter() + .map(|&lru| slru.list_item_num(segment, lru)) + .sum::(); + } + assert_eq!(items, slru.GetItemNum()); + assert!(slru.GetUsedSpace() <= slru.GetCapacity()); + } + + #[test] + fn parity_replacement_slru_maintainer_promotes_active_and_demotes_untouched() { + let mut slru = ReplacementSLRU::with_num_segments(100, 1); + slru.Init().unwrap(); + slru.TEST_ConfigLRUMaintainer(false); + slru.set_hot_lru_pct(20); + slru.set_warm_lru_pct(40); + assert_eq!(slru.segment_byte_limit(), 100); + + // Each entry occupies 1 key byte + 4 value bytes. + slru.Put(test_buffer("a", b"aaaa")); + assert!(slru.Get("a").is_some()); + assert!(slru.Get("a").is_some()); + assert_eq!(slru.TEST_CheckBufferFlag("a"), BUFFER_ACTIVE); + for key in ["b", "c", "d", "e"] { + slru.Put(test_buffer(key, b"xxxx")); + } + assert_eq!(slru.list_used_size(0, HOT_LRU), 25); + assert_eq!(slru.list_item_num(0, COLD_LRU), 0); + + // A disabled maintainer is a no-op even when the hot list is over its + // share of the shard budget. + assert!(slru.run_lru_maintainer_pass().is_empty()); + assert_eq!(slru.list_used_size(0, HOT_LRU), 25); + + // One pass trims the hot list to its 20% share (20 bytes). The tail is + // "a", touched twice, so it is promoted to warm with its flag reset. + slru.TEST_ConfigLRUMaintainer(true); + assert!(slru.run_lru_maintainer_pass().is_empty()); + assert_eq!(slru.TEST_CheckLRUPos("a"), WARM_LRU); + assert_eq!(slru.TEST_CheckBufferFlag("a"), BUFFER_INIT); + assert_eq!(slru.list_used_size(0, HOT_LRU), 20); + assert_eq!(slru.list_used_size(0, WARM_LRU), 5); + assert_eq!(slru.list_item_num(0, COLD_LRU), 0); + + // A second pass is a fixed point: nothing is over its share any more. + assert!(slru.LRUMaintainerTask().is_empty()); + assert_eq!(slru.list_used_size(0, HOT_LRU), 20); + assert_eq!(slru.list_used_size(0, WARM_LRU), 5); + + // Shrinking the shares drains hot, then warm, into the cold list. None + // of the entries were touched twice since the last pass, so they are + // demoted rather than promoted. + slru.set_hot_lru_pct(0); + slru.set_warm_lru_pct(0); + assert!(slru.run_lru_maintainer_pass().is_empty()); + assert_eq!(slru.list_item_num(0, HOT_LRU), 0); + assert_eq!(slru.list_item_num(0, WARM_LRU), 0); + assert_eq!(slru.list_item_num(0, COLD_LRU), 5); + assert_eq!(slru.list_used_size(0, COLD_LRU), 25); + assert_eq!(slru.GetUsedSpace(), 25); + assert_eq!(slru.GetItemNum(), 5); + } + + #[test] + fn parity_replacement_slru_maintainer_evicts_cold_tail_over_segment_budget() { + let mut slru = ReplacementSLRU::with_num_segments(40, 1); + slru.Init().unwrap(); + // Give hot and warm no share of the budget, so every insert is demoted + // into the cold list and reclaimed from there. + slru.set_hot_lru_pct(0); + slru.set_warm_lru_pct(0); + + let evicted_keys = Arc::new(std::sync::Mutex::new(Vec::new())); + let evicted_capture = Arc::clone(&evicted_keys); + slru.RegisterMemEvictionHandler(move |buffer| { + evicted_capture + .lock() + .unwrap() + .push(buffer.Key().to_string()); + }); + + // Four 10-byte entries exactly fill the single shard. + for key in ["a", "b", "c", "d"] { + assert!(slru.Put(test_buffer(key, b"xxxxxxxxx")).is_empty()); + } + assert_eq!(slru.GetUsedSpace(), 40); + assert_eq!(slru.list_item_num(0, COLD_LRU), 4); + assert_eq!(slru.list_item_num(0, HOT_LRU), 0); + + // The fifth insert puts the shard over budget; the cold tail is the + // oldest untouched entry and is handed to the lower tier before being + // dropped. + let evicted = slru.Put(test_buffer("e", b"xxxxxxxxx")); + let evicted_names: Vec = evicted + .iter() + .map(|buffer| buffer.Key().to_string()) + .collect(); + assert_eq!(evicted_names, vec!["a".to_string()]); + assert_eq!(*evicted_keys.lock().unwrap(), vec!["a".to_string()]); + assert_eq!(slru.GetUsedSpace(), 40); + assert_eq!(slru.GetItemNum(), 4); + assert!(slru.Peek("a").is_none()); + assert!(slru.Peek("e").is_some()); + } + + #[test] + fn parity_concurrent_slru_matches_the_single_threaded_segment_layout() { + let concurrent = ConcurrentReplacementSLRU::with_num_segments(1 << 16, 256); + let single = ReplacementSLRU::with_num_segments(1 << 16, 256); + assert_eq!(concurrent.num_segments(), single.num_segments()); + assert_eq!(concurrent.segment_byte_limit(), single.segment_byte_limit()); + assert_eq!(concurrent.GetCapacity(), 1 << 16); + + // A key lands in the same segment either way, so the two forms shard + // a workload identically. + for key in ["alpha", "beta", "gamma", "delta", "epsilon", "zeta"] { + assert_eq!(concurrent.segment_for_key(key), single.segment_for_key(key)); + } + + // Capacities below the segment count collapse to one segment in both. + assert_eq!(ConcurrentReplacementSLRU::new(6).num_segments(), 1); + assert_eq!(ReplacementSLRU::new(6).num_segments(), 1); + } + + #[test] + fn parity_concurrent_slru_serves_threads_through_per_segment_locks() { + let policy = ConcurrentReplacementSLRU::with_num_segments(1 << 16, 64); + policy.Init().unwrap(); + + // Four threads share the policy with no lock of their own. Keys that + // hash to different segments never contend. + std::thread::scope(|scope| { + for worker in 0..4 { + let policy = &policy; + scope.spawn(move || { + for index in 0..512 { + let key = format!("w{worker}-k{index:04}"); + policy.Put(test_buffer(&key, b"payload")); + let _ = policy.Get(&key); + if index % 3 == 0 { + let _ = policy.Delete(&key); + } + } + }); + } + }); + + // Every segment stayed inside its own budget and the totals reconcile, + // so no update was lost or double-counted across threads. + for segment in 0..policy.num_segments() { + assert!( + policy.segment_used_size(segment) <= policy.segment_byte_limit(), + "segment {segment} over budget" + ); + } + assert!(policy.GetUsedSpace() <= policy.GetCapacity()); + let summed: usize = (0..policy.num_segments()) + .map(|segment| policy.segment_item_num(segment)) + .sum(); + assert_eq!(summed, policy.GetItemNum()); + assert!(policy.GetItemNum() > 0); + } + + #[test] + fn parity_concurrent_slru_reports_evictions_from_every_segment() { + let policy = ConcurrentReplacementSLRU::with_num_segments(256, 4); + policy.Init().unwrap(); + let evicted = Arc::new(std::sync::Mutex::new(Vec::new())); + let capture = Arc::clone(&evicted); + policy.register_mem_eviction_handler(move |buffer| { + capture.lock().unwrap().push(buffer.Key().to_string()); + }); + + // 20 bytes per entry against a 64-byte segment budget forces eviction. + for index in 0..256 { + policy.Put(test_buffer(&format!("evict-{index:04}"), b"0123456789")); + } + + assert!(!evicted.lock().unwrap().is_empty()); + assert!(policy.GetUsedSpace() <= policy.GetCapacity()); + for segment in 0..policy.num_segments() { + assert!(policy.segment_used_size(segment) <= policy.segment_byte_limit()); + } + assert_eq!(policy.GetItemNum(), policy.GetItemNum()); + } + + #[test] + fn parity_concurrent_slru_round_trips_values_and_maintainer_passes() { + let policy = ConcurrentReplacementSLRU::with_num_segments(1 << 14, 8); + policy.Init().unwrap(); + policy.Put(test_buffer("round-trip", b"value")); + assert_eq!(policy.Get("round-trip").unwrap().Data(), b"value"); + assert_eq!(policy.Peek("round-trip").unwrap().Data(), b"value"); + + policy + .update_cache_buffer("round-trip", b"value", test_buffer("round-trip", b"next")) + .unwrap(); + assert_eq!(policy.Peek("round-trip").unwrap().Data(), b"next"); + assert!(matches!( + policy.update_cache_buffer("round-trip", b"stale", test_buffer("round-trip", b"bad")), + Err(CacheError::ReplaceMismatch) + )); + + // A maintainer sweep takes each segment lock in turn and is a no-op + // while nothing is over its share. + assert!(policy.LRUMaintainerTask().is_empty()); + assert_eq!(policy.GetItemNum(), 1); + + assert_eq!(policy.Delete("round-trip").unwrap().Key(), "round-trip"); + assert_eq!(policy.GetItemNum(), 0); + assert_eq!(policy.GetUsedSpace(), 0); + policy.Reset().unwrap(); + assert_eq!(policy.GetItemNum(), 0); + } + + fn order_key(index: usize) -> CacheKey { + CacheKey::string(0, &format!("order-key-{index:04}")) + } + + fn order_keys(order: &CacheKeyOrder) -> Vec { + order.iter().cloned().collect() + } + + #[test] + fn cache_key_order_tracks_recency_from_front_to_back() { + let mut order = CacheKeyOrder::new(); + assert!(order.is_empty()); + assert_eq!(order.front(), None); + assert_eq!(order.back(), None); + + for index in 0..4 { + order.push_back(order_key(index)); + } + assert_eq!(order.len(), 4); + assert_eq!(order.front(), Some(&order_key(0))); + assert_eq!(order.back(), Some(&order_key(3))); + + // A hit moves the key to the back and never duplicates it, which is + // what a plain deque needs a full rescan to guarantee. + assert!(order.touch(&order_key(0))); + assert_eq!(order.back(), Some(&order_key(0))); + assert_eq!(order.len(), 4); + assert_eq!( + order_keys(&order), + vec![order_key(1), order_key(2), order_key(3), order_key(0)] + ); + + // Touching the key that is already most recent is a no-op. + assert!(order.touch(&order_key(0))); + assert_eq!( + order_keys(&order), + vec![order_key(1), order_key(2), order_key(3), order_key(0)] + ); + + // Touching an absent key reports it and changes nothing. + assert!(!order.touch(&order_key(99))); + assert_eq!(order.len(), 4); + + // Eviction takes the least recently used first. + assert_eq!(order.pop_front(), Some(order_key(1))); + assert_eq!(order.pop_front(), Some(order_key(2))); + assert_eq!(order.len(), 2); + + assert!(order.remove(&order_key(3))); + assert!(!order.remove(&order_key(3))); + assert_eq!(order_keys(&order), vec![order_key(0)]); + + order.push_front(order_key(7)); + assert_eq!(order.front(), Some(&order_key(7))); + assert_eq!(order_keys(&order), vec![order_key(7), order_key(0)]); + + order.clear(); + assert!(order.is_empty()); + assert_eq!(order.pop_front(), None); + } + + #[test] + fn cache_key_order_walks_both_directions() { + let order: CacheKeyOrder = (0..4).map(order_key).collect(); + assert_eq!( + order.iter().cloned().collect::>(), + vec![ + order_key(0), + order_key(1), + order_key(2), + order_key(3) + ] + ); + // The reverse walk starts at the most recently used end. Eviction + // relies on this to reach the coldest entry first. + assert_eq!( + order.iter_rev().cloned().collect::>(), + vec![ + order_key(3), + order_key(2), + order_key(1), + order_key(0) + ] + ); + assert_eq!(order.iter_rev().next(), order.back()); + assert_eq!(order.iter().next(), order.front()); + + let empty = CacheKeyOrder::new(); + assert_eq!(empty.iter_rev().count(), 0); + } + + #[test] + fn simple_lru_evicts_the_coldest_entry_first() { + let cache = SimpleLRUCache::new(3 * 64); + let key = |name: &str| CacheKey::string(0, name); + let value = vec![118u8; 32]; + for name in ["a", "b", "c"] { + cache.Insert(key(name), value.clone(), 64).unwrap(); + } + + // Reading "a" makes "b" the coldest entry, so "b" is what the next + // insert must displace. + assert!(cache.Lookup(&key("a")).unwrap().is_some()); + cache.Insert(key("d"), value.clone(), 64).unwrap(); + + assert!( + cache.Lookup(&key("b")).unwrap().is_none(), + "the least recently used entry should be evicted" + ); + assert!( + cache.Lookup(&key("a")).unwrap().is_some(), + "a recently read entry must not be evicted" + ); + assert!(cache.Lookup(&key("c")).unwrap().is_some()); + assert!(cache.Lookup(&key("d")).unwrap().is_some()); + } + + #[test] + fn cache_key_order_retain_keeps_relative_order() { + let mut order: CacheKeyOrder = (0..8).map(order_key).collect(); + order.retain(|key| !key.record_key.ends_with('3') && !key.record_key.ends_with('5')); + assert_eq!( + order_keys(&order), + vec![ + order_key(0), + order_key(1), + order_key(2), + order_key(4), + order_key(6), + order_key(7), + ] + ); + assert_eq!(order.len(), 6); + assert!(!order.contains(&order_key(3))); + assert!(order.contains(&order_key(4))); + } + + #[test] + fn cache_key_order_matches_a_rescanning_deque_step_for_step() { + // The structure being replaced moved a key to the back by rescanning: + // if back() != Some(key) { retain(|c| c != key); push_back(key) } + // Drive both through the same operation stream and require the + // resulting recency order to agree after every single step, so the + // swap cannot change which entry gets evicted. + let mut order = CacheKeyOrder::new(); + let mut deque: VecDeque = VecDeque::new(); + + for step in 0..600usize { + let key = order_key(step.wrapping_mul(2_654_435_761) % 48); + match step % 5 { + 0..=2 => { + // A hit: only reorders a key that is already resident. + if deque.contains(&key) { + if deque.back() != Some(&key) { + deque.retain(|candidate| candidate != &key); + deque.push_back(key.clone()); + } + assert!(order.touch(&key)); + } else { + assert!(!order.touch(&key)); + } + } + 3 => { + // An insert of a key not yet resident. + if !deque.contains(&key) { + deque.push_back(key.clone()); + order.push_back(key.clone()); + } + } + _ => { + // A removal. + deque.retain(|candidate| candidate != &key); + order.remove(&key); + } + } + assert_eq!( + order_keys(&order), + deque.iter().cloned().collect::>(), + "diverged at step {step}" + ); + } + assert!(!order.is_empty()); + } + + #[test] + fn cache_key_order_recycles_nodes_across_churn() { + let mut order = CacheKeyOrder::new(); + for round in 0..6 { + for index in 0..64 { + order.push_back(order_key(round * 64 + index)); + } + assert_eq!(order.len(), 64); + while order.pop_front().is_some() {} + assert!(order.is_empty()); + } + order.push_back(order_key(1)); + order.push_back(order_key(2)); + assert_eq!(order_keys(&order), vec![order_key(1), order_key(2)]); + } + #[test] fn parity_hash_uint64_matches_matrixcache_vectors() { assert_eq!(hash_uint64(0), 0x5b03_af84_387a_42c6);