From 5a6461d10867f95ece1162f41fde31087a342f21 Mon Sep 17 00:00:00 2001 From: bjmeetsfo Date: Wed, 26 Aug 2026 17:54:03 -0700 Subject: [PATCH 1/3] a cache at capacity was weighing every resident key to evict one entry Once the memory tier is full, almost every write evicts, so whatever choosing a victim costs is paid on every write for the life of the cache. Choosing one looked like this: ```rust let keys = self.memory.keys().cloned().collect::>(); self.select_eviction_victim(keys) ``` That clones every resident key, scores all of them, groups them, and returns **one** victim -- and the caller runs it in a `while over_capacity` loop, so a write that has to free several entries pays that full sweep several times. The cost per write therefore climbs with the number of resident entries, which is exactly backwards: the fuller and busier the cache, the more each write costs. The fix weighs a bounded window of candidates instead, taken least-recently-accessed first, and falls back to the whole tier when the window turns up nothing evictable (all pinned), so a cache that can evict at all still evicts. Measured with `examples/eviction_bench.rs`, which reports both wall time per write and the number of candidate groups the selector formed per eviction. The second number is the one that settles the question: it is counted by the cache itself and is immune to load on the machine. ```text OLD -- weighs every resident key NEW -- bounded candidate window entries ns/write groups/eviction entries ns/write groups/eviction 1024 2436967 1025.0 1024 285566 512.0 2048 3288870 2049.0 2048 195288 512.0 4096 5863778 4097.0 4096 199734 512.0 8192 10409246 8193.0 8192 203738 512.0 hit rate, working set 4x the cache, 80% of reads on a hot half-cache 1024 82.37 % 1024 82.37 % 4096 81.99 % 4096 81.99 % ``` Groups weighed per eviction is `entries + 1` on the left and a flat 512 on the right: the old selector inspects the whole resident set, the new one inspects a fixed number of candidates however large the cache grows. Per write that is 12x at 1024 entries and 51x at 8192, and the line is still climbing on the left while it is flat on the right, so the distance keeps opening as a cache fills. The ladder stops at 8192 because the A/B has to run both arms at the same sizes and the old selector does not finish the larger ones in a sensible time -- which is the finding, not an inconvenience. Hit rate is identical to two decimals on both arms. That is the check that matters here, because a cheaper selector that quietly evicted better entries would show up as a faster cache that misses more. Two details that are load-bearing, and are why this is not simply "scan less": * The window walks **access** order, not insertion order. An earlier attempt at this walked insertion order and quietly cost about 11 points of hit rate, because the oldest-inserted entry is routinely the hottest one. * The fallback matters for correctness, not just for speed. With every candidate in the window pinned, a windowed-only selector reports "nothing to evict" and the tier grows past its capacity. Weighing the whole tier in that case costs no more than weighing everything did before, and picks exactly the victim the old code would have picked. The rest of this change brings the tree level with the sources these releases are cut from, which is how the selector above came to be sitting in one tree and not the other. Included there: `CacheKeyOrder` gains access tracking (`touch_access` / `iter_access`), the `*_fifo_order` fields become `*_order` now that they order more than FIFO, and `reset()` on the FIFO and segmented policies no longer clears the initialized flag -- resetting a cache emptied its index but left the policy refusing every later write while reporting success, which turned a reset cache into a black hole. Verified: 283 unit tests and the doc test pass, and `cargo check --all-targets` is clean against this crate's own manifest (rocksdb 0.25, thiserror 1) rather than the one the sources came from. --- examples/cache_scaling_bench.rs | 234 ++++ examples/eviction_bench.rs | 152 +++ examples/policy_bench.rs | 2 +- ...rity_bench.rs => rocksdb_backend_bench.rs} | 16 +- src/core/rdma.rs | 132 +- src/core/storage_config.rs | 405 +++--- src/lib.rs | 6 +- src/runtime/allocators_executors.rs | 78 +- src/runtime/builder_and_gc.rs | 247 ++-- src/runtime/cache_facades.rs | 221 +++- src/runtime/multilayer_cache.rs | 144 ++- src/runtime/replacement.rs | 35 +- src/runtime/storage_engines.rs | 28 +- src/tests/mod.rs | 1128 ++++++++++------- 14 files changed, 1926 insertions(+), 902 deletions(-) create mode 100755 examples/cache_scaling_bench.rs create mode 100755 examples/eviction_bench.rs rename examples/{rocksdb_parity_bench.rs => rocksdb_backend_bench.rs} (96%) diff --git a/examples/cache_scaling_bench.rs b/examples/cache_scaling_bench.rs new file mode 100755 index 0000000..ba486a8 --- /dev/null +++ b/examples/cache_scaling_bench.rs @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 MatrixArkAI + +//! Read-path cost and read concurrency for the multi-tier cache. +//! +//! Two things are worth watching on the read path, and they move +//! independently. +//! +//! The first is what a single memory-tier hit costs, and whether that cost +//! stays put as the resident set grows. A lookup that scans anything will show +//! up here as a number that climbs with the entry count. +//! +//! The second is what happens when several threads read at once. A cache that +//! serialises its readers gets *slower* per operation as threads are added +//! rather than faster in aggregate, and that shows up as throughput falling in +//! the scaling table below. Reads currently take the cache lock exclusively +//! because the hit updates statistics, hotness and latency histograms on the +//! way out, so this table is the measurement to beat when that changes. +//! +//! The third table puts `MultiLayerCache` next to `ShardedMultiLayerCache` +//! holding the same total capacity. The sharded cache spreads keys over +//! independent shards, so readers of different shards do not queue behind one +//! another. It is the supported answer to the contention in the second table, +//! and this comparison is what says whether it earns its extra bookkeeping. +//! +//! ```text +//! cargo run --release --example cache_scaling_bench +//! cargo run --release --example cache_scaling_bench -- 8192 +//! ``` + +use matrixcache::{CacheKey, CacheOptions, MultiLayerCache, ShardedMultiLayerCache}; +use std::time::{Duration, Instant}; + +const VALUE_BYTES: usize = 64; +const REPEATS: usize = 5; + +fn bench_dir(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("matrixcache-scaling-{name}")) +} + +/// Keys plus a visit order that touches each one but not in insertion order. +fn workload(entries: usize) -> Vec { + (0..entries) + .map(|index| CacheKey::string(0, &format!("scaling-key-{index:010}"))) + .collect() +} + +fn scattered(index: usize, len: usize) -> usize { + index.wrapping_mul(2_654_435_761) % len.max(1) +} + +fn ns_per_op(elapsed: Duration, ops: usize) -> f64 { + if ops == 0 { + return 0.0; + } + elapsed.as_nanos() as f64 / ops as f64 +} + +/// Cost of a single-threaded memory-tier hit at a given resident entry count. +fn hit_cost(entries: usize) -> f64 { + let dir = bench_dir(&format!("hit-{entries}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::new(entries * 256, &dir); + cache.start().expect("start cache"); + let keys = workload(entries); + let value = vec![b'v'; VALUE_BYTES]; + for key in &keys { + cache.put(key.clone(), value.clone()).expect("put"); + } + + let ops = entries * 8; + let mut best = f64::MAX; + for _ in 0..REPEATS { + let started = Instant::now(); + for index in 0..ops { + let _ = cache.get(&keys[scattered(index, entries)]).expect("get"); + } + best = best.min(ns_per_op(started.elapsed(), ops)); + } + let _ = std::fs::remove_dir_all(&dir); + best +} + +/// Aggregate read throughput with `threads` readers sharing one cache. +fn read_throughput(entries: usize, threads: usize) -> f64 { + let dir = bench_dir(&format!("conc-{threads}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::new(entries * 256, &dir); + cache.start().expect("start cache"); + let keys = workload(entries); + let value = vec![b'v'; VALUE_BYTES]; + for key in &keys { + cache.put(key.clone(), value.clone()).expect("put"); + } + + let per_thread = 40_000usize; + let mut best = f64::MAX; + for _ in 0..3 { + let started = Instant::now(); + std::thread::scope(|scope| { + for thread in 0..threads { + let cache = &cache; + let keys = &keys; + scope.spawn(move || { + for index in 0..per_thread { + // Offset per thread so readers are not in lockstep. + let slot = scattered(index + thread * 7919, keys.len()); + let _ = cache.get(&keys[slot]).expect("get"); + } + }); + } + }); + best = best.min(ns_per_op(started.elapsed(), per_thread * threads)); + } + let _ = std::fs::remove_dir_all(&dir); + best +} + +/// Options shared by both cache shapes, so the comparison is like for like. +fn scaling_options(dir: &std::path::Path, entries: usize) -> CacheOptions { + CacheOptions::new(entries * 256, 0, 0).with_ssd_paths(vec![dir.to_path_buf()]) +} + +/// Read throughput for the single-lock cache, built from shared options. +fn single_lock_throughput(entries: usize, threads: usize) -> f64 { + let dir = bench_dir(&format!("single-{threads}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::with_options(scaling_options(&dir, entries)); + cache.start().expect("start cache"); + let keys = workload(entries); + let value = vec![b'v'; VALUE_BYTES]; + for key in &keys { + cache.put(key.clone(), value.clone()).expect("put"); + } + let best = drive_readers(threads, &keys, |key| { + let _ = cache.get(key).expect("get"); + }); + let _ = std::fs::remove_dir_all(&dir); + best +} + +/// Read throughput for the sharded cache holding the same total capacity. +fn sharded_throughput(entries: usize, threads: usize, shards: usize) -> f64 { + let dir = bench_dir(&format!("sharded-{shards}-{threads}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = ShardedMultiLayerCache::with_options(scaling_options(&dir, entries), shards); + cache.start().expect("start cache"); + let keys = workload(entries); + let value = vec![b'v'; VALUE_BYTES]; + for key in &keys { + cache.put(key.clone(), value.clone()).expect("put"); + } + let best = drive_readers(threads, &keys, |key| { + let _ = cache.get(key).expect("get"); + }); + let _ = std::fs::remove_dir_all(&dir); + best +} + +/// Run `threads` readers over `keys` and return the best ns/op of three runs. +fn drive_readers(threads: usize, keys: &[CacheKey], read: F) -> f64 +where + F: Fn(&CacheKey) + Sync, +{ + let per_thread = 40_000usize; + let mut best = f64::MAX; + for _ in 0..3 { + let started = Instant::now(); + std::thread::scope(|scope| { + for thread in 0..threads { + let read = &read; + scope.spawn(move || { + for index in 0..per_thread { + // Offset per thread so readers are not in lockstep. + let slot = scattered(index + thread * 7919, keys.len()); + read(&keys[slot]); + } + }); + } + }); + best = best.min(ns_per_op(started.elapsed(), per_thread * threads)); + } + best +} + +fn main() { + let max_entries: usize = std::env::args() + .nth(1) + .and_then(|value| value.parse().ok()) + .unwrap_or(4_096); + + // Warm the allocator before the first measured case. + let _ = hit_cost(256); + + println!("memory-tier hit, single thread"); + println!("{:>10} {:>14}", "entries", "ns/op"); + let mut size = 1_024usize; + while size <= max_entries { + println!("{size:>10} {:>14.1}", hit_cost(size)); + size *= 4; + } + + println!(); + println!("read throughput, {max_entries} resident entries"); + println!("{:>10} {:>14} {:>14}", "threads", "ns/op", "Mops/s"); + for &threads in &[1usize, 2, 4, 8] { + let ns = read_throughput(max_entries, threads); + let mops = if ns > 0.0 { 1_000.0 / ns } else { 0.0 }; + println!("{threads:>10} {ns:>14.1} {mops:>14.2}"); + } + + println!(); + println!("single lock vs sharded, same total capacity, Mops/s"); + println!( + "{:>10} {:>14} {:>14} {:>10}", + "threads", "single", "sharded", "speedup" + ); + for &threads in &[1usize, 2, 4, 8] { + let single_ns = single_lock_throughput(max_entries, threads); + let sharded_ns = sharded_throughput(max_entries, threads, 16); + let single = if single_ns > 0.0 { + 1_000.0 / single_ns + } else { + 0.0 + }; + let sharded = if sharded_ns > 0.0 { + 1_000.0 / sharded_ns + } else { + 0.0 + }; + let speedup = if single > 0.0 { sharded / single } else { 0.0 }; + println!("{threads:>10} {single:>14.2} {sharded:>14.2} {speedup:>9.2}x"); + } +} diff --git a/examples/eviction_bench.rs b/examples/eviction_bench.rs new file mode 100755 index 0000000..1b7fdcb --- /dev/null +++ b/examples/eviction_bench.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 MatrixArkAI + +//! Cost of choosing an eviction victim, as the resident set grows. +//! +//! A cache at capacity evicts on almost every write, so whatever victim +//! selection costs is paid per write for the life of the cache. The thing to +//! watch is whether that cost stays put as the cache fills: a selector that +//! inspects every resident entry shows up here as a per-write cost that climbs +//! with the entry count, while one that inspects a bounded number of +//! candidates shows up as a flat line. +//! +//! Two numbers are reported per size. The first is wall time per write, which +//! is what a caller feels. The second is the number of candidate groups the +//! selector formed per evicted entry, which the cache already counts; it is +//! immune to load on the machine and is the number that says whether the +//! algorithm changed or only the weather did. +//! +//! ```text +//! cargo run --release --no-default-features --example eviction_bench +//! ``` + +use matrixcache::{CacheKey, CacheOptions, MultiLayerCache}; +use std::time::Instant; + +const VALUE_BYTES: usize = 64; +/// Room for the value plus its per-entry overhead, so `entries` really fit. +const SLOT_BYTES: usize = VALUE_BYTES; + +fn bench_dir(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("matrixcache-eviction-{name}")) +} + +fn key(index: usize) -> CacheKey { + CacheKey::string(0, &format!("eviction-key-{index:010}")) +} + +/// Spread successive steps over `len` slots so reads are not in insertion +/// order, which would flatter any policy that evicts from one end. +fn scattered(index: usize, len: usize) -> usize { + index.wrapping_mul(2_654_435_761) % len.max(1) +} + +/// Fill to capacity, then keep writing so every write evicts. +fn steady_state(entries: usize) -> (f64, f64) { + let dir = bench_dir(&format!("steady-{entries}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::with_options( + CacheOptions::new(entries * SLOT_BYTES, 0, 0).with_ssd_paths(vec![dir.clone()]), + ); + cache.start().expect("start cache"); + + let value = vec![b'v'; VALUE_BYTES]; + // Fill to capacity. Keys are built up front so the timed region below + // measures the cache rather than key formatting. + for index in 0..entries { + cache.put(key(index), value.clone()).expect("put"); + } + + let writes = 2_000usize; + let fresh: Vec = (entries..entries + writes).map(key).collect(); + + let before = cache.stats(); + let started = Instant::now(); + for k in &fresh { + cache.put(k.clone(), value.clone()).expect("put"); + } + let elapsed = started.elapsed(); + let after = cache.stats(); + + let evictions = after + .memory_evictions + .saturating_sub(before.memory_evictions); + let groups = after + .eviction_sampled_groups + .saturating_sub(before.eviction_sampled_groups); + + let ns_per_write = elapsed.as_nanos() as f64 / writes as f64; + let groups_per_eviction = if evictions == 0 { + 0.0 + } else { + groups as f64 / evictions as f64 + }; + + cache.stop(); + let _ = std::fs::remove_dir_all(&dir); + (ns_per_write, groups_per_eviction) +} + +/// Hit rate under a skewed read-through workload. +/// +/// Bounding the candidate search only pays off if it still throws out the +/// right entries. This drives a working set several times larger than the +/// cache, with most reads landing on a small hot subset, and reports the share +/// of reads the cache served. A selector that evicts hot entries shows up here +/// as a hit rate below what the hot subset alone would guarantee. +fn hit_rate(entries: usize) -> f64 { + let dir = bench_dir(&format!("hitrate-{entries}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::with_options( + CacheOptions::new(entries * SLOT_BYTES, 0, 0).with_ssd_paths(vec![dir.clone()]), + ); + cache.start().expect("start cache"); + + let value = vec![b'v'; VALUE_BYTES]; + // Four times as many keys as fit, with a hot subset that is half the + // cache, so a selector that protects hot entries can hold all of them. + let universe = entries * 4; + let hot = entries / 2; + let keys: Vec = (0..universe).map(key).collect(); + + let reads = 400_000usize; + let mut hits = 0usize; + for step in 0..reads { + // Four reads in five land in the hot subset; the rest sweep the + // universe and are the pressure that forces eviction. + let slot = if step % 5 < 4 { + scattered(step, hot) + } else { + scattered(step, universe) + }; + let k = &keys[slot]; + if cache.get(k).expect("get").is_some() { + hits += 1; + } else { + cache.put(k.clone(), value.clone()).expect("put"); + } + } + + cache.stop(); + let _ = std::fs::remove_dir_all(&dir); + hits as f64 * 100.0 / reads as f64 +} + +fn main() { + println!("steady-state write cost with the cache at capacity"); + println!( + "{:>10} {:>14} {:>22}", + "entries", "ns/write", "groups/eviction" + ); + for entries in [1_024usize, 2_048, 4_096, 8_192, 16_384, 32_768] { + let (ns, groups) = steady_state(entries); + println!("{entries:>10} {ns:>14.0} {groups:>22.1}"); + } + + println!(); + println!("hit rate, working set 4x the cache, 80% of reads on a hot half-cache"); + println!("{:>10} {:>14}", "entries", "hit rate %"); + for entries in [1_024usize, 4_096, 16_384] { + println!("{entries:>10} {:>14.2}", hit_rate(entries)); + } +} diff --git a/examples/policy_bench.rs b/examples/policy_bench.rs index 71d93e2..62705c7 100755 --- a/examples/policy_bench.rs +++ b/examples/policy_bench.rs @@ -5,7 +5,7 @@ //! //! Every policy keeps its keys in intrusive doubly-linked lists over a shared //! node arena, so a lookup or a delete unlinks an entry in constant time rather -//! than rescanning a list of keys. The segmented LRU additionally partitions +//! than rescanning a list of keys. The segmented Lru additionally partitions //! its index and lists into segments, each with its own byte budget, so //! eviction only ever walks a segment-local list. //! diff --git a/examples/rocksdb_parity_bench.rs b/examples/rocksdb_backend_bench.rs similarity index 96% rename from examples/rocksdb_parity_bench.rs rename to examples/rocksdb_backend_bench.rs index 831183c..7f7543e 100644 --- a/examples/rocksdb_parity_bench.rs +++ b/examples/rocksdb_backend_bench.rs @@ -122,7 +122,7 @@ fn parse_config() -> BenchConfig { } "--help" | "-h" => { println!( - "usage: rocksdb_parity_bench [iterations] [--iterations N] \ + "usage: rocksdb_backend_bench [iterations] [--iterations N] \ [--value-bytes N] [--dram-capacity-bytes N] \ [--pmem-capacity-bytes N] [--ssd-capacity-bytes N] \ [--placement-threshold-bytes N] \ @@ -279,7 +279,7 @@ fn main() { && restart_disk_refill_ready; println!("{{"); - println!(" \"report_version\": \"matrixcache_rocksdb_parity_v1\","); + println!(" \"report_version\": \"matrixcache_rocksdb_backend_v1\","); println!( " \"backend\": \"{}\",", if cfg!(feature = "rocksdb-ssd") { @@ -367,28 +367,28 @@ fn main() { println!(" \"matrixcache_contract_evidence\": {{"); println!(" \"dram_to_pmem_eviction\": {{"); println!(" \"observed\": {},", dram_to_pmem_eviction); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"memory_evictions > 0 && pmem_fills > 0\","); println!(" \"memory_evictions\": {},", stats.memory_evictions); println!(" \"pmem_fills\": {}", stats.pmem_fills); println!(" }},"); println!(" \"pmem_to_ssd_eviction\": {{"); println!(" \"observed\": {},", pmem_to_ssd_eviction); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"pmem_evictions > 0 && disk_fills > 0\","); println!(" \"pmem_evictions\": {},", stats.pmem_evictions); println!(" \"disk_fills\": {}", stats.disk_fills); println!(" }},"); println!(" \"ssd_read_through_refill\": {{"); println!(" \"observed\": {},", ssd_read_through_refill); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"cold_ssd_refills > 0 && refill_failures == 0\","); println!(" \"cold_ssd_refills\": {},", cold_ssd_refills); println!(" \"refill_failures\": {}", stats.refill_failures); println!(" }},"); println!(" \"replacement_soak\": {{"); println!(" \"observed\": {},", replacement_soak_ready); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"replacement_policy_soak.passed\","); println!(" \"iterations\": {},", soak_iterations); println!(" \"reasons\": {:?}", soak.reasons); @@ -398,7 +398,7 @@ fn main() { " \"observed\": {},", async_writeback_backpressure_ready ); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"observed_async_writeback_backpressure > 0\","); println!( " \"observed_async_writeback_backpressure\": {}", @@ -407,7 +407,7 @@ fn main() { println!(" }},"); println!(" \"restart_disk_refill\": {{"); println!(" \"observed\": {},", restart_disk_refill_ready); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"replacement_policy_soak.restart_disk_refill_ready\","); println!( " \"restart_disk_refill_ready\": {}", diff --git a/src/core/rdma.rs b/src/core/rdma.rs index 36bf1ae..c98d579 100644 --- a/src/core/rdma.rs +++ b/src/core/rdma.rs @@ -92,25 +92,35 @@ pub const CRC_LEN: usize = RDMA_CRC_LEN; pub const FAIL_ALLOC: i32 = RDMA_FAIL_ALLOC; #[allow(non_upper_case_globals)] pub const CRC_MISMATCH: i32 = RDMA_CRC_MISMATCH; - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum RdmaStorageEngineType { #[default] - DRAM = 0, - PMEM = 1, - SSD = 2, - INVALID = 3, + #[serde(alias = "DRAM")] + Dram = 0, + #[serde(alias = "PMEM")] + Pmem = 1, + #[serde(alias = "SSD")] + Ssd = 2, + #[serde(alias = "INVALID")] + Invalid = 3, +} + +#[allow(non_upper_case_globals)] +impl RdmaStorageEngineType { + pub const DRAM: Self = Self::Dram; + pub const PMEM: Self = Self::Pmem; + pub const SSD: Self = Self::Ssd; + pub const INVALID: Self = Self::Invalid; } impl RdmaStorageEngineType { pub fn from_code(code: u8) -> Self { match code { - 0 => Self::DRAM, - 1 => Self::PMEM, - 2 => Self::SSD, - _ => Self::INVALID, + 0 => Self::Dram, + 1 => Self::Pmem, + 2 => Self::Ssd, + _ => Self::Invalid, } } @@ -731,7 +741,7 @@ where continue; } let entry = &self.entries[pos]; - let signature_match = if entry.get_storage_engine_type() == RdmaStorageEngineType::SSD { + let signature_match = if entry.get_storage_engine_type() == RdmaStorageEngineType::Ssd { &entry.signature == sig128 } else { &entry.get_signature_96b() == sig96 @@ -802,7 +812,7 @@ where return RdmaHashTableGet { addr: None, len: 0, - storage_type: RdmaStorageEngineType::INVALID, + storage_type: RdmaStorageEngineType::Invalid, }; } let entry = &bucket.entries[pos as usize]; @@ -836,7 +846,7 @@ where status: RDMA_BUCKET_LOCKED, old_addr: None, old_len: 0, - old_type: RdmaStorageEngineType::INVALID, + old_type: RdmaStorageEngineType::Invalid, }; } let sig96 = signature_96(&key, bucket_pos as u64); @@ -845,7 +855,7 @@ where let block_size = kv_size.saturating_add(RDMA_DATA_HEADER + RDMA_CRC_LEN); let mut old_addr = None; let mut old_len = 0; - let mut old_type = RdmaStorageEngineType::INVALID; + let mut old_type = RdmaStorageEngineType::Invalid; let pos = if existing >= 0 { let pos = existing as usize; let old = &bucket.entries[pos]; @@ -867,7 +877,7 @@ where }; let entry = &mut bucket.entries[pos]; - if storage_type == RdmaStorageEngineType::SSD { + if storage_type == RdmaStorageEngineType::Ssd { entry.set_signature_128(sig128); } else { entry.set_signature_96(sig96); @@ -903,7 +913,7 @@ where status: RDMA_BUCKET_LOCKED, addr: None, len: 0, - storage_type: RdmaStorageEngineType::INVALID, + storage_type: RdmaStorageEngineType::Invalid, }; } let sig96 = signature_96(key, bucket_pos as u64); @@ -915,7 +925,7 @@ where status: RDMA_NOT_FOUND, addr: None, len: 0, - storage_type: RdmaStorageEngineType::INVALID, + storage_type: RdmaStorageEngineType::Invalid, }; } let entry = bucket.entries[pos as usize].clone(); @@ -1045,10 +1055,10 @@ pub struct RdmaStorageEngine { impl RdmaStorageEngine { pub fn new(storage_type: RdmaStorageEngineType, capacity: usize) -> Self { let base = match storage_type { - RdmaStorageEngineType::DRAM => 0x0100_0000, - RdmaStorageEngineType::PMEM => 0x0200_0000, - RdmaStorageEngineType::SSD => 0x0300_0000, - RdmaStorageEngineType::INVALID => 0x0400_0000, + RdmaStorageEngineType::Dram => 0x0100_0000, + RdmaStorageEngineType::Pmem => 0x0200_0000, + RdmaStorageEngineType::Ssd => 0x0300_0000, + RdmaStorageEngineType::Invalid => 0x0400_0000, }; Self { storage_type, @@ -1160,7 +1170,7 @@ pub struct RdmaStorageEngineDram { impl RdmaStorageEngineDram { pub fn new(capacity: usize) -> Self { Self { - inner: RdmaStorageEngine::new(RdmaStorageEngineType::DRAM, capacity), + inner: RdmaStorageEngine::new(RdmaStorageEngineType::Dram, capacity), } } @@ -1225,7 +1235,7 @@ pub struct RdmaStorageEnginePMem { impl RdmaStorageEnginePMem { pub fn new(capacity: usize) -> Self { Self { - inner: RdmaStorageEngine::new(RdmaStorageEngineType::PMEM, capacity), + inner: RdmaStorageEngine::new(RdmaStorageEngineType::Pmem, capacity), } } @@ -1300,7 +1310,7 @@ pub struct RdmaStorageEngineSSD { impl RdmaStorageEngineSSD { pub fn new(capacity: usize) -> Self { Self { - inner: RdmaStorageEngine::new(RdmaStorageEngineType::SSD, capacity), + inner: RdmaStorageEngine::new(RdmaStorageEngineType::Ssd, capacity), } } @@ -1361,18 +1371,28 @@ impl RdmaStorageEngineSSD { #[derive(Default)] pub enum RdmaReplacementPolicyType { #[default] - FIFO = 0, - LRU = 1, - OTHER = 2, + #[serde(alias = "FIFO")] + Fifo = 0, + #[serde(alias = "LRU")] + Lru = 1, + #[serde(alias = "OTHER")] + Other = 2, +} + +#[allow(non_upper_case_globals)] +impl RdmaReplacementPolicyType { + pub const FIFO: Self = Self::Fifo; + pub const LRU: Self = Self::Lru; + pub const OTHER: Self = Self::Other; } impl RdmaReplacementPolicyType { pub fn as_replacement_policy_type(self) -> ReplacementPolicyType { match self { - Self::FIFO => ReplacementPolicyType::kFIFO, - Self::LRU => ReplacementPolicyType::kLRU, - Self::OTHER => ReplacementPolicyType::kMaxCode, + Self::Fifo => ReplacementPolicyType::Fifo, + Self::Lru => ReplacementPolicyType::Lru, + Self::Other => ReplacementPolicyType::MaxCode, } } } @@ -1403,7 +1423,7 @@ impl RDMACache { } pub fn with_dram_capacity(dram_capacity: usize) -> Self { - Self::new(dram_capacity, 0, 0, RdmaReplacementPolicyType::FIFO) + Self::new(dram_capacity, 0, 0, RdmaReplacementPolicyType::Fifo) } pub fn lookup(&self, key: &[u8], response: &mut RDMAResponse) -> i32 { @@ -1413,22 +1433,22 @@ impl RDMACache { return RDMA_NOT_FOUND; }; match index.storage_type { - RdmaStorageEngineType::DRAM => { + RdmaStorageEngineType::Dram => { self.dram_engine.as_ref().map_or(RDMA_NOT_FOUND, |engine| { engine.get(key, index.len, response, addr) }) } - RdmaStorageEngineType::PMEM => { + RdmaStorageEngineType::Pmem => { self.pmem_engine.as_ref().map_or(RDMA_NOT_FOUND, |engine| { engine.get(key, index.len, response, addr) }) } - RdmaStorageEngineType::SSD => { + RdmaStorageEngineType::Ssd => { self.ssd_engine.as_ref().map_or(RDMA_NOT_FOUND, |engine| { engine.get(key, index.len, response, addr) }) } - RdmaStorageEngineType::INVALID => RDMA_NOT_FOUND, + RdmaStorageEngineType::Invalid => RDMA_NOT_FOUND, } } @@ -1438,7 +1458,7 @@ impl RDMACache { } pub fn insert(&mut self, key: &[u8], value: &[u8]) -> i32 { - self.insert_to_storage(RdmaStorageEngineType::DRAM, key, value) + self.insert_to_storage(RdmaStorageEngineType::Dram, key, value) } #[allow(non_snake_case)] @@ -1503,19 +1523,19 @@ impl RDMACache { pub fn get_capacity(&self, storage_type: RdmaStorageEngineType) -> usize { match storage_type { - RdmaStorageEngineType::DRAM => self + RdmaStorageEngineType::Dram => self .dram_engine .as_ref() .map_or(0, |engine| engine.stats().0), - RdmaStorageEngineType::PMEM => self + RdmaStorageEngineType::Pmem => self .pmem_engine .as_ref() .map_or(0, |engine| engine.stats().0), - RdmaStorageEngineType::SSD => self + RdmaStorageEngineType::Ssd => self .ssd_engine .as_ref() .map_or(0, |engine| engine.stats().0), - RdmaStorageEngineType::INVALID => 0, + RdmaStorageEngineType::Invalid => 0, } } @@ -1526,16 +1546,16 @@ impl RDMACache { pub fn init_storage_engine(&mut self, storage_type: RdmaStorageEngineType, capacity: usize) { match storage_type { - RdmaStorageEngineType::DRAM => { + RdmaStorageEngineType::Dram => { self.dram_engine = Some(RdmaStorageEngineDram::new(capacity)); } - RdmaStorageEngineType::PMEM => { + RdmaStorageEngineType::Pmem => { self.pmem_engine = Some(RdmaStorageEnginePMem::new(capacity)); } - RdmaStorageEngineType::SSD => { + RdmaStorageEngineType::Ssd => { self.ssd_engine = Some(RdmaStorageEngineSSD::new(capacity)); } - RdmaStorageEngineType::INVALID => {} + RdmaStorageEngineType::Invalid => {} } } @@ -1571,14 +1591,14 @@ impl RDMACache { storage_type: RdmaStorageEngineType, ) -> Option<(usize, usize, usize)> { match storage_type { - RdmaStorageEngineType::DRAM => { + RdmaStorageEngineType::Dram => { self.dram_engine.as_ref().map(RdmaStorageEngineDram::stats) } - RdmaStorageEngineType::PMEM => { + RdmaStorageEngineType::Pmem => { self.pmem_engine.as_ref().map(RdmaStorageEnginePMem::stats) } - RdmaStorageEngineType::SSD => self.ssd_engine.as_ref().map(RdmaStorageEngineSSD::stats), - RdmaStorageEngineType::INVALID => None, + RdmaStorageEngineType::Ssd => self.ssd_engine.as_ref().map(RdmaStorageEngineSSD::stats), + RdmaStorageEngineType::Invalid => None, } } @@ -1589,10 +1609,10 @@ impl RDMACache { value: &[u8], ) -> Option { match storage_type { - RdmaStorageEngineType::DRAM => self.dram_engine.as_mut()?.put(key, value), - RdmaStorageEngineType::PMEM => self.pmem_engine.as_mut()?.put(key, value), - RdmaStorageEngineType::SSD => self.ssd_engine.as_mut()?.put(key, value), - RdmaStorageEngineType::INVALID => None, + RdmaStorageEngineType::Dram => self.dram_engine.as_mut()?.put(key, value), + RdmaStorageEngineType::Pmem => self.pmem_engine.as_mut()?.put(key, value), + RdmaStorageEngineType::Ssd => self.ssd_engine.as_mut()?.put(key, value), + RdmaStorageEngineType::Invalid => None, } } @@ -1603,19 +1623,19 @@ impl RDMACache { len: usize, ) -> i32 { match storage_type { - RdmaStorageEngineType::DRAM => self + RdmaStorageEngineType::Dram => self .dram_engine .as_mut() .map_or(RDMA_NOT_FOUND, |engine| engine.del(addr, len)), - RdmaStorageEngineType::PMEM => self + RdmaStorageEngineType::Pmem => self .pmem_engine .as_mut() .map_or(RDMA_NOT_FOUND, |engine| engine.del(addr, len)), - RdmaStorageEngineType::SSD => self + RdmaStorageEngineType::Ssd => self .ssd_engine .as_mut() .map_or(RDMA_NOT_FOUND, |engine| engine.del(addr, len)), - RdmaStorageEngineType::INVALID => RDMA_NOT_FOUND, + RdmaStorageEngineType::Invalid => RDMA_NOT_FOUND, } } } diff --git a/src/core/storage_config.rs b/src/core/storage_config.rs index 64e469d..eb12ed9 100644 --- a/src/core/storage_config.rs +++ b/src/core/storage_config.rs @@ -12,84 +12,144 @@ pub struct CacheRecoverReport { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CacheTier { Memory, + #[serde(alias = "kPMEM")] Pmem, + #[serde(alias = "kSSD")] Ssd, Reject, } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum CacheInstanceType { - kDRAM = 0, - kPMEM = 1, - kSSD = 2, - kUnified = 3, + #[serde(alias = "kDRAM")] + Dram = 0, + #[serde(alias = "kPMEM")] + Pmem = 1, + #[serde(alias = "kSSD")] + Ssd = 2, + #[serde(alias = "kUnified")] + Unified = 3, +} + +#[allow(non_upper_case_globals)] +impl CacheInstanceType { + pub const kDRAM: Self = Self::Dram; + pub const kPMEM: Self = Self::Pmem; + pub const kSSD: Self = Self::Ssd; + pub const kUnified: Self = Self::Unified; } impl CacheInstanceType { fn as_tier(self) -> Option { match self { - CacheInstanceType::kDRAM => Some(CacheTier::Memory), - CacheInstanceType::kPMEM => Some(CacheTier::Pmem), - CacheInstanceType::kSSD => Some(CacheTier::Ssd), - CacheInstanceType::kUnified => None, + CacheInstanceType::Dram => Some(CacheTier::Memory), + CacheInstanceType::Pmem => Some(CacheTier::Pmem), + CacheInstanceType::Ssd => Some(CacheTier::Ssd), + CacheInstanceType::Unified => None, } } } - -#[allow(non_camel_case_types)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum StorageEngineType { - kDRAM, - kPMEM, - kSSD, - kSimple, - kMultiSSD, + #[serde(alias = "kDRAM")] + Dram, + #[serde(alias = "kPMEM")] + Pmem, + #[serde(alias = "kSSD")] + Ssd, + #[serde(alias = "kSimple")] + Simple, + #[serde(alias = "kMultiSSD")] + MultiSsd, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl StorageEngineType { + pub const kDRAM: Self = Self::Dram; + pub const kPMEM: Self = Self::Pmem; + pub const kSSD: Self = Self::Ssd; + pub const kSimple: Self = Self::Simple; + pub const kMultiSSD: Self = Self::MultiSsd; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum SSDEngineType { - /// Supported Rust SSD engine. - kRocksDB = 0, + /// Supported Rust Ssd engine. + #[serde(alias = "kRocksDB")] + RocksDb = 0, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl SSDEngineType { + pub const kRocksDB: Self = Self::RocksDb; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum WriteBufferType { - kUserDataBuf = 0, - kMetaDataBuf = 1, - kGCBuf = 2, - kCodecDataBuf = 3, + #[serde(alias = "kUserDataBuf")] + UserDataBuf = 0, + #[serde(alias = "kMetaDataBuf")] + MetaDataBuf = 1, + #[serde(alias = "kGCBuf")] + GcBuf = 2, + #[serde(alias = "kCodecDataBuf")] + CodecDataBuf = 3, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl WriteBufferType { + pub const kUserDataBuf: Self = Self::UserDataBuf; + pub const kMetaDataBuf: Self = Self::MetaDataBuf; + pub const kGCBuf: Self = Self::GcBuf; + pub const kCodecDataBuf: Self = Self::CodecDataBuf; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum DataType { - DATA = 1, - META_LOG = 2, + #[serde(alias = "DATA")] + Data = 1, + #[serde(alias = "META_LOG")] + MetaLog = 2, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl DataType { + pub const DATA: Self = Self::Data; + pub const META_LOG: Self = Self::MetaLog; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum GCMode { - LOSSY = 1, - LOSSLESS = 10, + #[serde(alias = "LOSSY")] + Lossy = 1, + #[serde(alias = "LOSSLESS")] + Lossless = 10, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl GCMode { + pub const LOSSY: Self = Self::Lossy; + pub const LOSSLESS: Self = Self::Lossless; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum RecordStateType { - kSoftDel = 0x0, - kNormal = 0x1, - kPinned = 0x2, - kMaxCode = 0xf, + #[serde(alias = "kSoftDel")] + SoftDel = 0x0, + #[serde(alias = "kNormal")] + Normal = 0x1, + #[serde(alias = "kPinned")] + Pinned = 0x2, + #[serde(alias = "kMaxCode")] + MaxCode = 0xf, +} + +#[allow(non_upper_case_globals)] +impl RecordStateType { + pub const kSoftDel: Self = Self::SoftDel; + pub const kNormal: Self = Self::Normal; + pub const kPinned: Self = Self::Pinned; + pub const kMaxCode: Self = Self::MaxCode; } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -155,8 +215,8 @@ impl SsdIndex { let mut entries = self.entries.write().expect("ssd index lock poisoned"); let value = entries.get_mut(key)?; if let SsdIndexValue::Memory { state, .. } = value { - if *state == RecordStateType::kSoftDel { - *state = RecordStateType::kNormal; + if *state == RecordStateType::SoftDel { + *state = RecordStateType::Normal; } } Some(value.clone()) @@ -165,8 +225,8 @@ impl SsdIndex { pub fn unpin(&self, key: &str) { let mut entries = self.entries.write().expect("ssd index lock poisoned"); if let Some(SsdIndexValue::Memory { state, .. }) = entries.get_mut(key) { - if *state == RecordStateType::kPinned { - *state = RecordStateType::kNormal; + if *state == RecordStateType::Pinned { + *state = RecordStateType::Normal; } } } @@ -176,18 +236,18 @@ impl SsdIndex { let Some(SsdIndexValue::Memory { state, .. }) = entries.get_mut(key) else { return false; }; - if *state == RecordStateType::kPinned { + if *state == RecordStateType::Pinned { return false; } - *state = RecordStateType::kPinned; + *state = RecordStateType::Pinned; true } pub fn soft_delete(&self, key: &str) { let mut entries = self.entries.write().expect("ssd index lock poisoned"); if let Some(SsdIndexValue::Memory { state, .. }) = entries.get_mut(key) { - if *state != RecordStateType::kPinned { - *state = RecordStateType::kSoftDel; + if *state != RecordStateType::Pinned { + *state = RecordStateType::SoftDel; } } } @@ -417,7 +477,7 @@ impl WriteBuffer { impl Default for WriteBuffer { fn default() -> Self { - Self::new(WriteBufferType::kUserDataBuf, 10_485_760) + Self::new(WriteBufferType::UserDataBuf, 10_485_760) } } @@ -522,7 +582,7 @@ pub fn mask_colored_ptr_size(old_colored_ptr: u64, size: u32) -> u64 { } pub fn mask_colored_ptr_record_state(old_colored_ptr: u64, state: RecordStateType) -> u64 { - old_colored_ptr | ((state as u64) & (RecordStateType::kMaxCode as u64)) + old_colored_ptr | ((state as u64) & (RecordStateType::MaxCode as u64)) } #[allow(non_snake_case)] @@ -718,7 +778,7 @@ impl BufferEncoder { let record_size = Self::DATA_FIXED_PART_SIZE.saturating_add(value_len); let record_units = aligned_to(record_size, self.align_size) / self.align_size as u32; let mut colored_ptr = 0; - colored_ptr = mask_colored_ptr_record_state(colored_ptr, RecordStateType::kSoftDel); + colored_ptr = mask_colored_ptr_record_state(colored_ptr, RecordStateType::SoftDel); colored_ptr = mask_colored_ptr_lba(colored_ptr, batch_begin_offset); colored_ptr = mask_colored_ptr_size(colored_ptr, record_units); update_entry_cb(&record.key, SsdIndexValue::SsdColoredPtr(colored_ptr)); @@ -978,44 +1038,44 @@ impl BufferManager { } impl SSDEngineType { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("rocksdb") || value.eq_ignore_ascii_case("rocks_db") || value.eq_ignore_ascii_case("kRocksDB") || value.eq_ignore_ascii_case("kSSDRocksDBStorageEngine") { - Self::kRocksDB + Self::RocksDb } else { - Self::kRocksDB + Self::RocksDb } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kRocksDB => "RocksDB", + Self::RocksDb => "RocksDB", } } #[allow(non_snake_case)] - pub fn FromReferenceName(value: &str) -> Self { - Self::from_reference_name(value) + pub fn FromConfigName(value: &str) -> Self { + Self::from_config_name(value) } #[allow(non_snake_case)] - pub fn AsReferenceName(self) -> &'static str { - self.as_reference_name() + pub fn AsConfigName(self) -> &'static str { + self.as_config_name() } } impl StorageEngineType { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("pmem") || value.eq_ignore_ascii_case("persistent_memory") || value.eq_ignore_ascii_case("persistent-memory") || value.eq_ignore_ascii_case("kPMEMStorageEngine") || value.eq_ignore_ascii_case("kPMEM") { - Self::kPMEM + Self::Pmem } else if value.eq_ignore_ascii_case("ssd") || value.eq_ignore_ascii_case("rocksdb") || value.eq_ignore_ascii_case("rocks_db") @@ -1023,76 +1083,76 @@ impl StorageEngineType { || value.eq_ignore_ascii_case("kSSDRocksDBStorageEngine") || value.eq_ignore_ascii_case("kSSD") { - Self::kSSD + Self::Ssd } else if value.eq_ignore_ascii_case("simple") || value.eq_ignore_ascii_case("simple_storage") || value.eq_ignore_ascii_case("kSimpleStorageEngine") { - Self::kSimple + Self::Simple } else if value.eq_ignore_ascii_case("multi_ssd") || value.eq_ignore_ascii_case("multi-ssd") || value.eq_ignore_ascii_case("kMultiSSDStorageEngine") { - Self::kMultiSSD + Self::MultiSsd } else if value.eq_ignore_ascii_case("dram") || value.eq_ignore_ascii_case("kDRAM") || value.eq_ignore_ascii_case("kDRAMStorageEngine") { - Self::kDRAM + Self::Dram } else { - Self::kDRAM + Self::Dram } } - pub fn from_reference_code(value: u8) -> Self { + pub fn from_config_code(value: u8) -> Self { match value { - 1 => Self::kPMEM, - 2 => Self::kSSD, - 3 => Self::kSimple, - 4 => Self::kMultiSSD, - _ => Self::kDRAM, + 1 => Self::Pmem, + 2 => Self::Ssd, + 3 => Self::Simple, + 4 => Self::MultiSsd, + _ => Self::Dram, } } - pub fn reference_code(self) -> u8 { + pub fn config_code(self) -> u8 { match self { - Self::kDRAM => 0, - Self::kPMEM => 1, - Self::kSSD => 2, - Self::kSimple => 3, - Self::kMultiSSD => 4, + Self::Dram => 0, + Self::Pmem => 1, + Self::Ssd => 2, + Self::Simple => 3, + Self::MultiSsd => 4, } } pub fn is_ssd_like(self) -> bool { - matches!(self, Self::kSSD | Self::kMultiSSD) + matches!(self, Self::Ssd | Self::MultiSsd) } pub fn canonical_instance_type(self) -> CacheInstanceType { match self { - Self::kDRAM | Self::kSimple => CacheInstanceType::kDRAM, - Self::kPMEM => CacheInstanceType::kPMEM, - Self::kSSD | Self::kMultiSSD => CacheInstanceType::kSSD, + Self::Dram | Self::Simple => CacheInstanceType::Dram, + Self::Pmem => CacheInstanceType::Pmem, + Self::Ssd | Self::MultiSsd => CacheInstanceType::Ssd, } } - pub fn as_reference_enum_name(self) -> &'static str { + pub fn as_config_enum_name(self) -> &'static str { match self { - Self::kDRAM => "kDRAMStorageEngine", - Self::kPMEM => "kPMEMStorageEngine", - Self::kSSD => "kSSDRocksDBStorageEngine", - Self::kSimple => "kSimpleStorageEngine", - Self::kMultiSSD => "kMultiSSDStorageEngine", + Self::Dram => "kDRAMStorageEngine", + Self::Pmem => "kPMEMStorageEngine", + Self::Ssd => "kSSDRocksDBStorageEngine", + Self::Simple => "kSimpleStorageEngine", + Self::MultiSsd => "kMultiSSDStorageEngine", } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kDRAM => "DRAM", - Self::kPMEM => "PMEM", - Self::kSSD => "SSD", - Self::kSimple => "Simple", - Self::kMultiSSD => "MultiSSD", + Self::Dram => "DRAM", + Self::Pmem => "PMEM", + Self::Ssd => "SSD", + Self::Simple => "Simple", + Self::MultiSsd => "MultiSSD", } } @@ -1101,13 +1161,13 @@ impl StorageEngineType { } #[allow(non_snake_case)] - pub fn FromReferenceCode(value: u8) -> Self { - Self::from_reference_code(value) + pub fn FromConfigCode(value: u8) -> Self { + Self::from_config_code(value) } #[allow(non_snake_case)] - pub fn ReferenceCode(self) -> u8 { - self.reference_code() + pub fn ConfigCode(self) -> u8 { + self.config_code() } #[allow(non_snake_case)] @@ -1116,73 +1176,87 @@ impl StorageEngineType { } #[allow(non_snake_case)] - pub fn AsReferenceEnumName(self) -> &'static str { - self.as_reference_enum_name() + pub fn AsConfigEnumName(self) -> &'static str { + self.as_config_enum_name() } #[allow(non_snake_case)] - pub fn AsReferenceName(self) -> &'static str { - self.as_reference_name() + pub fn AsConfigName(self) -> &'static str { + self.as_config_name() } } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum ReplacementPolicyType { - kFIFO = 0, - kLRU = 1, - kSLRU = 2, - kWeightedHotnessLru = 3, - kMaxCode = 4, + #[serde(alias = "kFIFO")] + Fifo = 0, + #[serde(alias = "kLRU")] + Lru = 1, + #[serde(alias = "kSLRU")] + Slru = 2, + #[serde(alias = "kWeightedHotnessLru")] + WeightedHotnessLru = 3, + #[serde(alias = "kMaxCode")] + MaxCode = 4, +} + +#[allow(non_upper_case_globals)] +impl ReplacementPolicyType { + pub const kFIFO: Self = Self::Fifo; + pub const kLRU: Self = Self::Lru; + pub const kSLRU: Self = Self::Slru; + pub const kWeightedHotnessLru: Self = Self::WeightedHotnessLru; + pub const kMaxCode: Self = Self::MaxCode; } impl ReplacementPolicyType { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("fifo") || value.eq_ignore_ascii_case("kFIFO") { - Self::kFIFO + Self::Fifo } else if value.eq_ignore_ascii_case("slru") || value.eq_ignore_ascii_case("kSLRU") { - Self::kSLRU + Self::Slru } else if value.eq_ignore_ascii_case("lru") || value.eq_ignore_ascii_case("kLRU") { - Self::kLRU + Self::Lru } else if value.eq_ignore_ascii_case("kMaxCode") { - Self::kMaxCode + Self::MaxCode } else { - Self::kWeightedHotnessLru + Self::WeightedHotnessLru } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kFIFO => "FIFO", - Self::kSLRU => "SLRU", - Self::kLRU => "LRU", - Self::kWeightedHotnessLru => "WeightedHotnessLru", - Self::kMaxCode => "MaxCode", + Self::Fifo => "FIFO", + Self::Slru => "SLRU", + Self::Lru => "LRU", + Self::WeightedHotnessLru => "WeightedHotnessLru", + Self::MaxCode => "MaxCode", } } fn as_cache_policy(self) -> CacheReplacementPolicy { match self { - ReplacementPolicyType::kFIFO => CacheReplacementPolicy::Fifo, - ReplacementPolicyType::kSLRU => CacheReplacementPolicy::Slru, - ReplacementPolicyType::kLRU => CacheReplacementPolicy::WeightedHotnessLru, - ReplacementPolicyType::kWeightedHotnessLru => { + ReplacementPolicyType::Fifo => CacheReplacementPolicy::Fifo, + ReplacementPolicyType::Slru => CacheReplacementPolicy::Slru, + ReplacementPolicyType::Lru => CacheReplacementPolicy::WeightedHotnessLru, + ReplacementPolicyType::WeightedHotnessLru => { CacheReplacementPolicy::WeightedHotnessLru } - ReplacementPolicyType::kMaxCode => CacheReplacementPolicy::WeightedHotnessLru, + ReplacementPolicyType::MaxCode => CacheReplacementPolicy::WeightedHotnessLru, } } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CacheDataPlacement { + #[serde(alias = "kSideBySide")] SideBySide, + #[serde(alias = "kTiered")] Tiered, } impl CacheDataPlacement { - pub fn try_from_reference_name(value: &str) -> Result { + pub fn try_from_config_name(value: &str) -> Result { if value.eq_ignore_ascii_case("sidebyside") || value.eq_ignore_ascii_case("side_by_side") || value.eq_ignore_ascii_case("side-by-side") @@ -1202,7 +1276,7 @@ impl CacheDataPlacement { } } - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("sidebyside") || value.eq_ignore_ascii_case("side_by_side") || value.eq_ignore_ascii_case("side-by-side") @@ -1213,75 +1287,83 @@ impl CacheDataPlacement { } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { Self::SideBySide => "SideBySide", Self::Tiered => "Tiered", } } } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum DRAMPMEMDataPlacementType { - kSideBySide = 0, - kTiered = 1, - kMaxCode = 2, + #[serde(alias = "kSideBySide")] + SideBySide = 0, + #[serde(alias = "kTiered")] + Tiered = 1, + #[serde(alias = "kMaxCode")] + MaxCode = 2, +} + +#[allow(non_upper_case_globals)] +impl DRAMPMEMDataPlacementType { + pub const kSideBySide: Self = Self::SideBySide; + pub const kTiered: Self = Self::Tiered; + pub const kMaxCode: Self = Self::MaxCode; } impl DRAMPMEMDataPlacementType { - pub fn try_from_reference_name(value: &str) -> Result { - Ok(match CacheDataPlacement::try_from_reference_name(value)? { - CacheDataPlacement::SideBySide => Self::kSideBySide, - CacheDataPlacement::Tiered => Self::kTiered, + pub fn try_from_config_name(value: &str) -> Result { + Ok(match CacheDataPlacement::try_from_config_name(value)? { + CacheDataPlacement::SideBySide => Self::SideBySide, + CacheDataPlacement::Tiered => Self::Tiered, }) } - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("sidebyside") || value.eq_ignore_ascii_case("side_by_side") || value.eq_ignore_ascii_case("side-by-side") || value.eq_ignore_ascii_case("kSideBySide") { - Self::kSideBySide + Self::SideBySide } else if value.eq_ignore_ascii_case("kMaxCode") { - Self::kMaxCode + Self::MaxCode } else { - Self::kTiered + Self::Tiered } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kSideBySide => "SideBySide", - Self::kTiered => "Tiered", - Self::kMaxCode => "MaxCode", + Self::SideBySide => "SideBySide", + Self::Tiered => "Tiered", + Self::MaxCode => "MaxCode", } } pub fn as_cache_data_placement(self) -> CacheDataPlacement { match self { - Self::kSideBySide => CacheDataPlacement::SideBySide, - Self::kTiered | Self::kMaxCode => CacheDataPlacement::Tiered, + Self::SideBySide => CacheDataPlacement::SideBySide, + Self::Tiered | Self::MaxCode => CacheDataPlacement::Tiered, } } pub fn from_cache_data_placement(placement: CacheDataPlacement) -> Self { match placement { - CacheDataPlacement::SideBySide => Self::kSideBySide, - CacheDataPlacement::Tiered => Self::kTiered, + CacheDataPlacement::SideBySide => Self::SideBySide, + CacheDataPlacement::Tiered => Self::Tiered, } } #[allow(non_snake_case)] - pub fn FromReferenceName(value: &str) -> Self { - Self::from_reference_name(value) + pub fn FromConfigName(value: &str) -> Self { + Self::from_config_name(value) } #[allow(non_snake_case)] - pub fn AsReferenceName(self) -> &'static str { - self.as_reference_name() + pub fn AsConfigName(self) -> &'static str { + self.as_config_name() } #[allow(non_snake_case)] @@ -1320,8 +1402,11 @@ pub enum CacheAdmissionReason { #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CacheAccessRecordType { + #[serde(alias = "kPut")] Put = 1, + #[serde(alias = "kGet")] Get = 2, + #[serde(alias = "kDelete")] Delete = 3, } @@ -1332,11 +1417,11 @@ impl CacheAccessRecordType { pub const kDelete: Self = Self::Delete; pub const kMaxCode: u8 = 4; - pub fn reference_code(self) -> u8 { + pub fn config_code(self) -> u8 { self as u8 } - pub fn from_reference_code(code: u8) -> Option { + pub fn from_config_code(code: u8) -> Option { match code { 1 => Some(Self::Put), 2 => Some(Self::Get), @@ -1345,7 +1430,7 @@ impl CacheAccessRecordType { } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { Self::Put => "kPut", Self::Get => "kGet", @@ -1354,18 +1439,18 @@ impl CacheAccessRecordType { } #[allow(non_snake_case)] - pub fn ReferenceCode(self) -> u8 { - self.reference_code() + pub fn ConfigCode(self) -> u8 { + self.config_code() } #[allow(non_snake_case)] - pub fn FromReferenceCode(code: u8) -> Option { - Self::from_reference_code(code) + pub fn FromConfigCode(code: u8) -> Option { + Self::from_config_code(code) } #[allow(non_snake_case)] - pub fn AsReferenceName(self) -> &'static str { - self.as_reference_name() + pub fn AsConfigName(self) -> &'static str { + self.as_config_name() } } diff --git a/src/lib.rs b/src/lib.rs index fd724bf..408ad11 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,7 @@ // Copyright 2026 MatrixArkAI //! MatrixCache is a Rust-native multi-tier cache library. It manages a hot -//! in-memory (DRAM) tier, a persistent-memory-like resident tier, and an SSD tier +//! in-memory (Dram) tier, a persistent-memory-like resident tier, and an Ssd tier //! (RocksDB by default), with admission control, cross-tier eviction, read-through //! refill, pinned handles, invalidation, and asynchronous writeback with //! backpressure accounting. @@ -12,7 +12,7 @@ //! ```no_run //! use matrixcache::{CacheKey, MultiLayerCache}; //! -//! // 1 MiB in-memory tier; the SSD tier is persisted under `dir`. +//! // 1 MiB in-memory tier; the Ssd tier is persisted under `dir`. //! let dir = std::env::temp_dir().join("matrixcache-doc"); //! let cache = MultiLayerCache::new(1 << 20, dir); //! @@ -22,7 +22,7 @@ //! # Ok::<(), matrixcache::CacheError>(()) //! ``` //! -//! The SSD backend is RocksDB via the default `rocksdb-ssd` feature; build with +//! The Ssd backend is RocksDB via the default `rocksdb-ssd` feature; build with //! `--no-default-features` for a lightweight file-backed compatibility store. use std::collections::hash_map::DefaultHasher; diff --git a/src/runtime/allocators_executors.rs b/src/runtime/allocators_executors.rs index 3f176e1..e03dec0 100644 --- a/src/runtime/allocators_executors.rs +++ b/src/runtime/allocators_executors.rs @@ -33,47 +33,57 @@ impl AllocatorStats { self.num_occupied_bytes } } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum AllocatorType { - kLogBasedAllocator = 0, - kPoolBasedAllocator = 1, - kJeAllocator = 2, - kMaxCode = 3, + #[serde(alias = "kLogBasedAllocator")] + LogBasedAllocator = 0, + #[serde(alias = "kPoolBasedAllocator")] + PoolBasedAllocator = 1, + #[serde(alias = "kJeAllocator")] + JeAllocator = 2, + #[serde(alias = "kMaxCode")] + MaxCode = 3, +} + +#[allow(non_upper_case_globals)] +impl AllocatorType { + pub const kLogBasedAllocator: Self = Self::LogBasedAllocator; + pub const kPoolBasedAllocator: Self = Self::PoolBasedAllocator; + pub const kJeAllocator: Self = Self::JeAllocator; + pub const kMaxCode: Self = Self::MaxCode; } impl AllocatorType { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("log") || value.eq_ignore_ascii_case("log_based") || value.eq_ignore_ascii_case("logbased") || value.eq_ignore_ascii_case("kLogBasedAllocator") { - Self::kLogBasedAllocator + Self::LogBasedAllocator } else if value.eq_ignore_ascii_case("pool") || value.eq_ignore_ascii_case("pool_based") || value.eq_ignore_ascii_case("poolbased") || value.eq_ignore_ascii_case("kPoolBasedAllocator") { - Self::kPoolBasedAllocator + Self::PoolBasedAllocator } else if value.eq_ignore_ascii_case("je") || value.eq_ignore_ascii_case("jemalloc") || value.eq_ignore_ascii_case("kJeAllocator") { - Self::kJeAllocator + Self::JeAllocator } else { - Self::kMaxCode + Self::MaxCode } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kLogBasedAllocator => "LogBasedAllocator", - Self::kPoolBasedAllocator => "PoolBasedAllocator", - Self::kJeAllocator => "JeAllocator", - Self::kMaxCode => "MaxCode", + Self::LogBasedAllocator => "LogBasedAllocator", + Self::PoolBasedAllocator => "PoolBasedAllocator", + Self::JeAllocator => "JeAllocator", + Self::MaxCode => "MaxCode", } } } @@ -91,38 +101,46 @@ pub struct PoolChunkMeta { pub id: ChunkID, pub num_alloc_objects: usize, } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum FlushPolicy { - kNoFlush = 0, - kInstantFlush = 1, - kMiniBatchFlush = 2, + #[serde(alias = "kNoFlush")] + NoFlush = 0, + #[serde(alias = "kInstantFlush")] + InstantFlush = 1, + #[serde(alias = "kMiniBatchFlush")] + MiniBatchFlush = 2, +} + +#[allow(non_upper_case_globals)] +impl FlushPolicy { + pub const kNoFlush: Self = Self::NoFlush; + pub const kInstantFlush: Self = Self::InstantFlush; + pub const kMiniBatchFlush: Self = Self::MiniBatchFlush; } impl FlushPolicy { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("no_flush") || value.eq_ignore_ascii_case("noflush") || value.eq_ignore_ascii_case("kNoFlush") { - Self::kNoFlush + Self::NoFlush } else if value.eq_ignore_ascii_case("instant_flush") || value.eq_ignore_ascii_case("instant") || value.eq_ignore_ascii_case("kInstantFlush") { - Self::kInstantFlush + Self::InstantFlush } else { - Self::kMiniBatchFlush + Self::MiniBatchFlush } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kNoFlush => "NoFlush", - Self::kInstantFlush => "InstantFlush", - Self::kMiniBatchFlush => "MiniBatchFlush", + Self::NoFlush => "NoFlush", + Self::InstantFlush => "InstantFlush", + Self::MiniBatchFlush => "MiniBatchFlush", } } } @@ -213,7 +231,7 @@ fn free_virtual_region(ptr: AllocatorPtr) -> Result, CacheError> { } pub fn parse_allocator_type(allocator_type: &str) -> AllocatorType { - AllocatorType::from_reference_name(allocator_type) + AllocatorType::from_config_name(allocator_type) } pub fn dram_allocate_object( diff --git a/src/runtime/builder_and_gc.rs b/src/runtime/builder_and_gc.rs index f6723d9..1fe4246 100644 --- a/src/runtime/builder_and_gc.rs +++ b/src/runtime/builder_and_gc.rs @@ -689,7 +689,7 @@ impl CacheInner { } } self.pmem.clear(); - self.pmem_fifo_order.clear(); + self.pmem_order.clear(); self.pmem_bytes = 0; for (key, expected_len) in live { report.scanned_files = report.scanned_files.saturating_add(1); @@ -761,7 +761,7 @@ impl CacheInner { }); } self.disk_index = recovered_index; - self.disk_fifo_order = recovered_order.into_iter().collect(); + self.disk_order = recovered_order.into_iter().collect(); self.ssd_bytes = recovered_bytes; self.stats.disk_bytes = recovered_bytes; Ok(report) @@ -836,7 +836,7 @@ impl CacheInner { } self.disk_index = recovered_index; - self.disk_fifo_order = recovered_order.into_iter().collect(); + self.disk_order = recovered_order.into_iter().collect(); self.ssd_bytes = recovered_bytes; self.stats.disk_bytes = recovered_bytes; Ok(report) @@ -904,28 +904,52 @@ impl CacheInner { fn record_hit_metadata(&mut self, key: &CacheKey, block_bytes: usize) { self.access_epoch = self.access_epoch.saturating_add(1); - let block_kind = infer_block_kind(key); - let entry = self.metadata.entry(key.clone()).or_insert(CacheEntryMeta { - block_kind, - routing_slot: extract_routing_slot(key), - hotness: initial_hotness(block_kind, block_bytes), - hits: 0, - last_access_epoch: 0, - admission_reason: CacheAdmissionReason::MemoryOnly, - }); - entry.hits = entry.hits.saturating_add(1); - let before = entry.hotness; - entry.hotness = entry.hotness.saturating_add(1); - entry.last_access_epoch = self.access_epoch; - if before < self.tiering_policy.memory_hotness_threshold - && entry.hotness >= self.tiering_policy.memory_hotness_threshold - { + let epoch = self.access_epoch; + let threshold = self.tiering_policy.memory_hotness_threshold; + + // The hit path runs for every read, so it must not pay for the miss + // path. Looking the entry up first keeps the key clone, the block-kind + // inference and the routing-slot extraction on the branch that + // actually needs them; going through `entry()` did all of that on + // every hit and then discarded it. + let crossed_threshold = if let Some(entry) = self.metadata.get_mut(key) { + entry.hits = entry.hits.saturating_add(1); + let before = entry.hotness; + entry.hotness = entry.hotness.saturating_add(1); + entry.last_access_epoch = epoch; + before < threshold && entry.hotness >= threshold + } else { + let block_kind = infer_block_kind(key); + let before = initial_hotness(block_kind, block_bytes); + let hotness = before.saturating_add(1); + self.metadata.insert( + key.clone(), + CacheEntryMeta { + block_kind, + routing_slot: extract_routing_slot(key), + hotness, + hits: 1, + last_access_epoch: epoch, + admission_reason: CacheAdmissionReason::MemoryOnly, + }, + ); + before < threshold && hotness >= threshold + }; + + if crossed_threshold { self.stats.hotness_promotions = self.stats.hotness_promotions.saturating_add(1); } } fn record_hit(&mut self, key: &CacheKey, block_bytes: usize) { self.record_hit_metadata(key, block_bytes); + // Move the entry to the back of each tier's access order. Victim + // selection reads the front of that order, so without this an entry + // written early stays at the front however often it is read, and is + // offered up for eviction on every pass. + self.memory_order.touch_access(key); + self.pmem_order.touch_access(key); + self.disk_order.touch_access(key); } fn put_memory(&mut self, key: CacheKey, value: Vec) -> bool { @@ -941,7 +965,7 @@ impl CacheInner { if let Some(old) = self.memory.insert(key.clone(), Arc::clone(&value)) { self.memory_bytes = self.memory_bytes.saturating_sub(old.len()); } else { - self.memory_fifo_order.push_back_if_absent(key); + self.memory_order.push_back_if_absent(key); } self.memory_bytes += value.len(); self.evict_memory_to_capacity_since(eviction_started); @@ -977,7 +1001,7 @@ impl CacheInner { if let Some(old) = self.pmem.insert(key.clone(), Arc::clone(&value)) { self.pmem_bytes = self.pmem_bytes.saturating_sub(old.len()); } else { - self.pmem_fifo_order.push_back_if_absent(key); + self.pmem_order.push_back_if_absent(key); } self.pmem_bytes = self.pmem_bytes.saturating_add(value.len()); self.evict_pmem_to_capacity_since(eviction_started); @@ -1021,7 +1045,7 @@ impl CacheInner { return false; } self.disk_index.insert(key.clone(), block_len as u64); - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len as u64); self.stats.disk_fills = self.stats.disk_fills.saturating_add(1); self.stats.ssd_admission_accepted = self.stats.ssd_admission_accepted.saturating_add(1); @@ -1079,7 +1103,6 @@ impl CacheInner { } fn evict_memory_to_capacity_since(&mut self, eviction_started: Instant) { - let mut victim_keys = Vec::new(); while self.memory_bytes > self.memory_capacity_bytes { let before = self.memory_bytes; let Some((victim, reason, pinned_skips)) = self.select_memory_eviction_victim() else { @@ -1091,11 +1114,14 @@ impl CacheInner { .stats .eviction_pinned_skips .saturating_add(pinned_skips); + // Drop the victim from the order as it is taken. Selection reads + // that order, so leaving it until after the loop lets the next + // round pick the same key, whose bytes are already gone. + self.memory_order.remove(&victim); let Some(old_value) = self.memory.remove(&victim) else { - victim_keys.push(victim); - if self.memory_bytes == before { - break; - } + // A key the order still listed but the tier no longer holds. + // It is gone from the order now, so the next round makes + // progress rather than picking it again. continue; }; self.memory_bytes = self.memory_bytes.saturating_sub(old_value.len()); @@ -1111,23 +1137,16 @@ impl CacheInner { { self.metadata.remove(&victim); } - victim_keys.push(victim); self.record_eviction_latency(eviction_started); if self.memory_bytes == before { break; } } - if !victim_keys.is_empty() { - let victim_key_set = victim_keys.iter().cloned().collect::>(); - self.memory_fifo_order - .retain(|candidate| !victim_key_set.contains(candidate)); - } self.stats.memory_bytes = self.memory_bytes as u64; self.refresh_pin_stats(); } fn evict_pmem_to_capacity_since(&mut self, eviction_started: Instant) { - let mut victim_keys = Vec::new(); while self.pmem_bytes > self.pmem_capacity_bytes { let before = self.pmem_bytes; let Some((victim, _reason, pinned_skips)) = self.select_pmem_eviction_victim() else { @@ -1139,11 +1158,14 @@ impl CacheInner { .stats .pmem_eviction_pinned_skips .saturating_add(pinned_skips); + // Drop the victim from the order as it is taken. Selection reads + // that order, so leaving it until after the loop lets the next + // round pick the same key, whose bytes are already gone. + self.pmem_order.remove(&victim); let Some(old_value) = self.pmem.remove(&victim) else { - victim_keys.push(victim); - if self.pmem_bytes == before { - break; - } + // A key the order still listed but the tier no longer holds. + // It is gone from the order now, so the next round makes + // progress rather than picking it again. continue; }; self.pmem_bytes = self.pmem_bytes.saturating_sub(old_value.len()); @@ -1158,17 +1180,11 @@ impl CacheInner { { self.metadata.remove(&victim); } - victim_keys.push(victim); self.record_eviction_latency(eviction_started); if self.pmem_bytes == before { break; } } - if !victim_keys.is_empty() { - let victim_key_set = victim_keys.iter().cloned().collect::>(); - self.pmem_fifo_order - .retain(|candidate| !victim_key_set.contains(candidate)); - } self.stats.pmem_bytes = self.pmem_bytes as u64; self.refresh_pin_stats(); } @@ -1197,7 +1213,7 @@ impl CacheInner { // Drop the victim from the order as it is taken. Selection reads // that order, so leaving it until after the loop lets the next // round pick the same key, whose bytes are already gone. - self.disk_fifo_order.remove(&victim); + self.disk_order.remove(&victim); self.record_eviction(CacheTier::Ssd, victim.clone(), evicted_value.clone()); self.ssd_bytes = self.ssd_bytes.saturating_sub(removed_bytes); self.stats.ssd_evictions = self.stats.ssd_evictions.saturating_add(1); @@ -1218,9 +1234,6 @@ impl CacheInner { self.stats.disk_bytes = self.ssd_bytes; return; } - let victim_key_set = victim_keys.iter().cloned().collect::>(); - self.disk_fifo_order - .retain(|candidate| !victim_key_set.contains(candidate)); let _ = self.delete_ssd_blocks(&victim_keys); for key in &victim_keys { let _ = self.append_disk_manifest_delete(key); @@ -1231,11 +1244,12 @@ impl CacheInner { fn select_memory_eviction_victim(&mut self) -> Option<(CacheKey, EvictionReason, u64)> { match self.memory_replacement_policy { CacheReplacementPolicy::Fifo => { - self.select_fifo_eviction_victim(&self.memory_fifo_order) + self.select_fifo_eviction_victim(&self.memory_order) } CacheReplacementPolicy::Slru | CacheReplacementPolicy::WeightedHotnessLru => { - let keys = self.memory.keys().cloned().collect::>(); - self.select_eviction_victim(keys) + let picked = self.select_windowed_eviction_victim(&self.memory_order); + self.record_sampled_groups(picked.groups_weighed); + picked.victim } } } @@ -1243,11 +1257,12 @@ impl CacheInner { fn select_pmem_eviction_victim(&mut self) -> Option<(CacheKey, EvictionReason, u64)> { match self.pmem_replacement_policy { CacheReplacementPolicy::Fifo => { - self.select_fifo_eviction_victim(&self.pmem_fifo_order) + self.select_fifo_eviction_victim(&self.pmem_order) } CacheReplacementPolicy::Slru | CacheReplacementPolicy::WeightedHotnessLru => { - let keys = self.pmem.keys().cloned().collect::>(); - self.select_eviction_victim(keys) + let picked = self.select_windowed_eviction_victim(&self.pmem_order); + self.record_sampled_groups(picked.groups_weighed); + picked.victim } } } @@ -1255,11 +1270,12 @@ impl CacheInner { fn select_ssd_eviction_victim(&mut self) -> Option<(CacheKey, EvictionReason, u64)> { match self.ssd_replacement_policy { CacheReplacementPolicy::Fifo => { - self.select_fifo_eviction_victim(&self.disk_fifo_order) + self.select_fifo_eviction_victim(&self.disk_order) } CacheReplacementPolicy::Slru | CacheReplacementPolicy::WeightedHotnessLru => { - let keys = self.disk_index.keys().cloned().collect::>(); - self.select_eviction_victim(keys) + let picked = self.select_windowed_eviction_victim(&self.disk_order); + self.record_sampled_groups(picked.groups_weighed); + picked.victim } } } @@ -1279,29 +1295,60 @@ impl CacheInner { None } - fn select_eviction_victim(&mut self, keys: I) -> Option<(CacheKey, EvictionReason, u64)> + /// Weigh a bounded window of `order`, least recently accessed first. + /// + /// Falls back to the whole tier when the window turns up nothing + /// evictable, so a run of pinned entries at the front of the order cannot + /// stall eviction while unpinned entries sit further back. A tier holding + /// no more than the window weighs everything either way, so its victim is + /// exactly the one it would have picked before the window existed. + fn select_windowed_eviction_victim(&self, order: &CacheKeyOrder) -> PickedEvictionVictim { + let windowed = + self.select_eviction_victim(order.iter_access().take(EVICTION_CANDIDATE_WINDOW)); + if windowed.victim.is_some() || order.len() <= EVICTION_CANDIDATE_WINDOW { + return windowed; + } + let full = self.select_eviction_victim(order.iter_access()); + PickedEvictionVictim { + victim: full.victim, + groups_weighed: windowed + .groups_weighed + .saturating_add(full.groups_weighed), + } + } + + fn record_sampled_groups(&mut self, groups: usize) { + self.stats.eviction_sampled_groups = self + .stats + .eviction_sampled_groups + .saturating_add(groups as u64); + } + + /// Weigh `keys` and return the coldest group's coldest member. + /// + /// Borrows the keys rather than taking them by value: the caller passes the + /// tier's own map keys straight in, so a selection no longer starts by + /// cloning every key in the tier. + fn select_eviction_victim<'a, I>(&self, keys: I) -> PickedEvictionVictim where - I: IntoIterator, + I: IntoIterator, { let mut pinned_skips = 0u64; - let mut groups: HashMap = HashMap::new(); + let mut groups: HashMap, SlotEvictionGroup> = HashMap::new(); for key in keys { - if self.pinned.contains_key(&key) { + if self.pinned.contains_key(key) { pinned_skips = pinned_skips.saturating_add(1); continue; } - let score = self.eviction_score(&key); - let group_key = self.eviction_group_key(&key); + let score = self.eviction_score(key); + let group_key = self.eviction_group_key(key); groups .entry(group_key) - .and_modify(|group| group.observe(key.clone(), score)) + .and_modify(|group| group.observe(key, score)) .or_insert_with(|| SlotEvictionGroup::new(key, score)); } - self.stats.eviction_sampled_groups = self - .stats - .eviction_sampled_groups - .saturating_add(groups.len() as u64); - groups + let group_count = groups.len(); + let victim = groups .into_values() .min_by(|left, right| { left.group_score @@ -1315,7 +1362,11 @@ impl CacheInner { eviction_reason_for(group.victim_score), pinned_skips, ) - }) + }); + PickedEvictionVictim { + victim, + groups_weighed: group_count, + } } fn eviction_score(&self, key: &CacheKey) -> EvictionScore { @@ -1348,10 +1399,11 @@ impl CacheInner { let incoming_group = request .routing_slot .or_else(|| extract_routing_slot(key)) - .map(|slot| format!("slot:{slot}")) - .unwrap_or_else(|| format!("object:{}:{}", key.namespace, key.record_key)); - self.disk_index - .keys() + .map(EvictionGroupKey::Slot) + .unwrap_or(EvictionGroupKey::Object(&key.namespace, &key.record_key)); + self.disk_order + .iter() + .take(EVICTION_CANDIDATE_WINDOW) .filter(|candidate| self.eviction_group_key(candidate) != incoming_group) .map(|candidate| self.eviction_score(candidate)) .min() @@ -1359,13 +1411,13 @@ impl CacheInner { .unwrap_or(false) } - fn eviction_group_key(&self, key: &CacheKey) -> String { + fn eviction_group_key<'a>(&self, key: &'a CacheKey) -> EvictionGroupKey<'a> { self.metadata .get(key) .and_then(|meta| meta.routing_slot) .or_else(|| extract_routing_slot(key)) - .map(|slot| format!("slot:{slot}")) - .unwrap_or_else(|| format!("object:{}:{}", key.namespace, key.record_key)) + .map(EvictionGroupKey::Slot) + .unwrap_or(EvictionGroupKey::Object(&key.namespace, &key.record_key)) } fn record_memory_eviction_reason(&mut self, reason: EvictionReason) { @@ -1452,7 +1504,7 @@ impl CacheInner { removed_pinned_bytes = removed_pinned_bytes.max(value.len()); self.memory_bytes = self.memory_bytes.saturating_sub(value.len()); } - self.memory_fifo_order.remove(key); + self.memory_order.remove(key); if remove_disk { let disk_bytes = self.disk_index.remove(key).unwrap_or_default(); removed_pinned_bytes = removed_pinned_bytes.max( @@ -1479,7 +1531,7 @@ impl CacheInner { if remove_disk { let _ = self.persist_pmem_delete(key); } - self.pmem_fifo_order.remove(key); + self.pmem_order.remove(key); self.metadata.remove(key); if key_pinned && removed_pinned_bytes > 0 { self.pinned_removed_bytes @@ -1493,11 +1545,11 @@ impl CacheInner { if remove_disk { match key_set.as_ref() { Some(key_set) => { - self.disk_fifo_order + self.disk_order .retain(|candidate| !key_set.contains(candidate)); } None => { - self.disk_fifo_order + self.disk_order .retain(|candidate| !keys.contains(candidate)); } } @@ -1515,9 +1567,9 @@ impl CacheInner { self.pmem.clear(); self.clear_pmem_persistence()?; self.disk_index.clear(); - self.disk_fifo_order.clear(); - self.memory_fifo_order.clear(); - self.pmem_fifo_order.clear(); + self.disk_order.clear(); + self.memory_order.clear(); + self.pmem_order.clear(); self.pinned.clear(); self.pinned_handle_bytes.clear(); self.pinned_removed_bytes.clear(); @@ -1625,7 +1677,11 @@ impl CacheInner { } fn record_get_latency(&mut self, started: Instant) { - let micros = started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + let micros = elapsed_micros(started); + self.record_get_latency_micros(micros); + } + + fn record_get_latency_micros(&mut self, micros: u64) { self.stats.get_latency_samples = self.stats.get_latency_samples.saturating_add(1); self.stats.get_latency_total_micros = self.stats.get_latency_total_micros.saturating_add(micros); @@ -1661,7 +1717,11 @@ impl CacheInner { } fn record_read_through_latency(&mut self, started: Instant) { - let micros = started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + let micros = elapsed_micros(started); + self.record_read_through_latency_micros(micros); + } + + fn record_read_through_latency_micros(&mut self, micros: u64) { observe_latency_bucket( micros, &mut self.stats.read_through_latency_samples, @@ -1781,6 +1841,16 @@ fn infer_block_kind(key: &CacheKey) -> CacheBlockKind { } } +/// How many candidates victim selection weighs before it settles. +/// +/// Selection used to weigh every resident entry, so a cache sitting at +/// capacity paid a cost proportional to how much it held on every single +/// write. Weighing a bounded window of the oldest-resident entries instead +/// keeps that cost flat as the cache grows. The window is wider than the +/// working set of a small cache, so those keep weighing everything and choose +/// exactly what they chose before. +const EVICTION_CANDIDATE_WINDOW: usize = 512; + fn eviction_reason_for(score: EvictionScore) -> EvictionReason { if score.hotness == 0 { EvictionReason::Cold @@ -1873,3 +1943,8 @@ fn unique_temp_path(kind: &str) -> PathBuf { std::process::id() )) } + +/// Microseconds since , saturating rather than wrapping. +fn elapsed_micros(started: Instant) -> u64 { + started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64 +} diff --git a/src/runtime/cache_facades.rs b/src/runtime/cache_facades.rs index 2746653..12c7c87 100644 --- a/src/runtime/cache_facades.rs +++ b/src/runtime/cache_facades.rs @@ -197,15 +197,15 @@ impl CacheInstance { .cloned() .unwrap_or_else(|| unique_temp_path("cache-instance")); let mut options = match storage_type { - StorageEngineType::kDRAM | StorageEngineType::kSimple => { + StorageEngineType::Dram | StorageEngineType::Simple => { CacheOptions::new(capacity, 0, 0) } - StorageEngineType::kPMEM => CacheOptions::new(0, capacity, 0), - StorageEngineType::kSSD | StorageEngineType::kMultiSSD => { + StorageEngineType::Pmem => CacheOptions::new(0, capacity, 0), + StorageEngineType::Ssd | StorageEngineType::MultiSsd => { CacheOptions::new(0, 0, capacity).with_ssd_instance_only(true) } }; - let ssd_paths = if matches!(storage_type, StorageEngineType::kSSD | StorageEngineType::kMultiSSD) + let ssd_paths = if matches!(storage_type, StorageEngineType::Ssd | StorageEngineType::MultiSsd) && !paths.is_empty() { paths.clone() @@ -216,7 +216,7 @@ impl CacheInstance { instance_type.as_tier().expect("storage-backed instance"), replacement_type.as_cache_policy(), ); - if matches!(storage_type, StorageEngineType::kPMEM) { + if matches!(storage_type, StorageEngineType::Pmem) { options = options.with_pmem_paths(paths); } @@ -478,8 +478,8 @@ impl CacheInstance { pub fn recover_data(&self) -> Result { match self.storage_type { - StorageEngineType::kPMEM => self.cache.recover_pmem_index(), - StorageEngineType::kSSD | StorageEngineType::kMultiSSD => self.cache.recover_disk_index(), + StorageEngineType::Pmem => self.cache.recover_pmem_index(), + StorageEngineType::Ssd | StorageEngineType::MultiSsd => self.cache.recover_disk_index(), _ => Ok(CacheRecoverReport::default()), } } @@ -533,12 +533,12 @@ impl CacheInstance { pub fn get_allocator_type(&self) -> AllocatorType { match self.storage_type { - StorageEngineType::kDRAM | StorageEngineType::kSimple => { - AllocatorType::kPoolBasedAllocator + StorageEngineType::Dram | StorageEngineType::Simple => { + AllocatorType::PoolBasedAllocator } - StorageEngineType::kPMEM => AllocatorType::kLogBasedAllocator, - StorageEngineType::kSSD | StorageEngineType::kMultiSSD => { - AllocatorType::kLogBasedAllocator + StorageEngineType::Pmem => AllocatorType::LogBasedAllocator, + StorageEngineType::Ssd | StorageEngineType::MultiSsd => { + AllocatorType::LogBasedAllocator } } } @@ -1631,7 +1631,10 @@ impl SimpleLRUCache { #[allow(non_snake_case)] pub fn Size(&self) -> usize { - self.inner.lock().expect("simple lru lock poisoned").size + self.inner + .lock() + .expect("simple lru lock poisoned") + .current_size() } } @@ -1756,7 +1759,10 @@ impl ZeroCopySimpleLRUCache { #[allow(non_snake_case)] pub fn Size(&self) -> usize { - self.inner.lock().expect("simple lru lock poisoned").size + self.inner + .lock() + .expect("simple lru lock poisoned") + .current_size() } #[allow(non_snake_case)] @@ -1893,6 +1899,13 @@ struct SimpleLruInner { size: usize, order: CacheKeyOrder, entries: HashMap, + /// Entries removed while a handle still pinned them. + /// + /// Their bytes are still resident, so they keep counting towards the + /// cache size until the last pin is released. Dropping them from the + /// accounting at removal would let the cache admit data it has no room + /// for. + pinned_removed: Vec, } impl SimpleLruInner { @@ -1906,6 +1919,7 @@ impl SimpleLruInner { size: 0, order: CacheKeyOrder::new(), entries: HashMap::new(), + pinned_removed: Vec::new(), } } @@ -1914,6 +1928,7 @@ impl SimpleLruInner { if size > self.capacity { return false; } + self.reap_pinned_removed(); self.remove(&key); self.size += size; self.order.push_front(key.clone()); @@ -1930,15 +1945,46 @@ impl SimpleLruInner { fn remove(&mut self, key: &CacheKey) { if let Some(entry) = self.entries.remove(key) { - self.size = self.size.saturating_sub(entry.size); self.order.remove(key); + self.retire(entry); } } fn remove_all(&mut self) { - self.entries.clear(); + let retired = self.entries.drain().map(|(_, entry)| entry).collect::>(); + for entry in retired { + self.retire(entry); + } self.order.clear(); - self.size = 0; + } + + /// Drop an entry from the index, keeping its bytes counted while a handle + /// still holds the value. + fn retire(&mut self, entry: SimpleLruEntry) { + if Arc::strong_count(&entry.value) > 1 { + self.pinned_removed.push(entry); + } else { + self.size = self.size.saturating_sub(entry.size); + } + } + + /// Release the bytes of any removed entry whose last pin has now dropped. + fn reap_pinned_removed(&mut self) { + let mut released = 0usize; + self.pinned_removed.retain(|entry| { + if Arc::strong_count(&entry.value) > 1 { + return true; + } + released = released.saturating_add(entry.size); + false + }); + self.size = self.size.saturating_sub(released); + } + + /// Current size, after accounting for pins released since the last call. + fn current_size(&mut self) -> usize { + self.reap_pinned_removed(); + self.size } fn set_capacity(&mut self, capacity: usize) { @@ -1957,6 +2003,7 @@ impl SimpleLruInner { } fn evict_unpinned(&mut self) { + self.reap_pinned_removed(); while self.size > self.capacity { // Walk from the least recently used end and stop at the first // entry nobody is holding. Scanning forward for the last match @@ -2342,14 +2389,14 @@ impl FlexibleCache { pmem_paths: impl IntoIterator, ssd_paths: impl IntoIterator, ) -> Self { - let policy = ReplacementPolicyType::from_reference_name(policy.as_ref()); - let engine = StorageEngineType::from_reference_name(engine.as_ref()); + let policy = ReplacementPolicyType::from_config_name(policy.as_ref()); + let engine = StorageEngineType::from_config_name(engine.as_ref()); let paths = match engine { - StorageEngineType::kPMEM => pmem_paths.into_iter().collect::>(), - StorageEngineType::kSSD | StorageEngineType::kMultiSSD => { + StorageEngineType::Pmem => pmem_paths.into_iter().collect::>(), + StorageEngineType::Ssd | StorageEngineType::MultiSsd => { ssd_paths.into_iter().collect::>() } - StorageEngineType::kDRAM | StorageEngineType::kSimple => Vec::new(), + StorageEngineType::Dram | StorageEngineType::Simple => Vec::new(), }; Self { instance: CacheInstance::new(capacity, policy, engine, paths.clone()), @@ -2677,11 +2724,11 @@ impl MultiTierCache { side_by_side_dram_pmem_placement_threshold: usize, ssd_storage_engine: impl AsRef, ) -> Result { - let policy_type = ReplacementPolicyType::from_reference_name(policy.as_ref()); + let policy_type = ReplacementPolicyType::from_config_name(policy.as_ref()); let replacement_policy = policy_type.as_cache_policy(); - let ssd_storage_engine = StorageEngineType::from_reference_name(ssd_storage_engine.as_ref()); + let ssd_storage_engine = StorageEngineType::from_config_name(ssd_storage_engine.as_ref()); let data_placement = - CacheDataPlacement::try_from_reference_name(dram_pmem_data_placement_type.as_ref())?; + CacheDataPlacement::try_from_config_name(dram_pmem_data_placement_type.as_ref())?; let options = CacheOptions::new(dram_capacity, pmem_capacity, ssd_capacity) .with_pmem_paths(pmem_paths) .with_ssd_paths(ssd_paths) @@ -2826,9 +2873,9 @@ impl MultiTierCache { "matrixcache_stats metrics={} comments={} policy={} ssd_engine={} placement={} eviction_enabled={} memory_bytes={} pmem_bytes={} disk_bytes={} pinned_entries={} pinned_bytes={} memory_evictions={} pmem_evictions={} ssd_evictions={} ssd_admission_rejections={} async_writeback_queue_depth={} async_writeback_queue_bytes={} writeback_backpressure_events={}", metrics.as_ref(), comments.as_ref(), - self.policy.as_reference_name(), - self.ssd_storage_engine.as_reference_name(), - self.cache.production_tiering_policy().data_placement.as_reference_name(), + self.policy.as_config_name(), + self.ssd_storage_engine.as_config_name(), + self.cache.production_tiering_policy().data_placement.as_config_name(), self.eviction_enabled(), stats.memory_bytes, stats.pmem_bytes, @@ -2977,6 +3024,8 @@ struct CacheOrderNode { key: CacheKey, prev: u32, next: u32, + access_prev: u32, + access_next: u32, } /// Recency ordering over cache keys, from least recently used at the front to @@ -2997,6 +3046,8 @@ pub struct CacheKeyOrder { index: HashMap, head: u32, tail: u32, + access_head: u32, + access_tail: u32, } impl Default for CacheKeyOrder { @@ -3013,6 +3064,8 @@ impl CacheKeyOrder { index: HashMap::new(), head: CACHE_ORDER_NIL, tail: CACHE_ORDER_NIL, + access_head: CACHE_ORDER_NIL, + access_tail: CACHE_ORDER_NIL, } } @@ -3049,10 +3102,13 @@ impl CacheKeyOrder { if let Some(&node) = self.index.get(&key) { self.unlink(node); self.link_back(node); + self.unlink_access(node); + self.link_access_back(node); return; } let node = self.alloc(key.clone()); self.link_back(node); + self.link_access_back(node); self.index.insert(key, node); } @@ -3068,6 +3124,7 @@ impl CacheKeyOrder { } let node = self.alloc(key.clone()); self.link_back(node); + self.link_access_back(node); self.index.insert(key, node); } @@ -3076,16 +3133,22 @@ impl CacheKeyOrder { if let Some(&node) = self.index.get(&key) { self.unlink(node); self.link_front(node); + self.unlink_access(node); + self.link_access_front(node); return; } let node = self.alloc(key.clone()); self.link_front(node); + self.link_access_front(node); self.index.insert(key, node); } - /// Record a hit: move `key` to the back if it is present. Returns whether - /// the key was there. - pub fn touch(&mut self, key: &CacheKey) -> bool { + /// Move `key` to the back of the insertion order if it is present. + /// Returns whether the key was there. + /// + /// This is not how a hit is recorded — a hit must leave the insertion + /// order alone, which is what `touch_access` is for. + pub fn move_to_back(&mut self, key: &CacheKey) -> bool { let Some(&node) = self.index.get(key) else { return false; }; @@ -3097,12 +3160,33 @@ impl CacheKeyOrder { true } + /// Record an access: move `key` to the back of the access order, leaving + /// the insertion order alone. Returns whether the key was there. + /// + /// The two orders answer different questions. Eviction that only ever + /// looks at the front of a list needs the front to hold entries nobody + /// wants; insertion order never moves an entry no matter how often it is + /// read, so a popular entry written early sits at the front forever and is + /// offered up on every pass. + pub fn touch_access(&mut self, key: &CacheKey) -> bool { + let Some(&node) = self.index.get(key) else { + return false; + }; + if node == self.access_tail { + return true; + } + self.unlink_access(node); + self.link_access_back(node); + true + } + /// Remove `key`, returning whether it was present. pub fn remove(&mut self, key: &CacheKey) -> bool { let Some(node) = self.index.remove(key) else { return false; }; self.unlink(node); + self.unlink_access(node); self.release(node); true } @@ -3115,6 +3199,7 @@ impl CacheKeyOrder { let node = self.head; let key = self.nodes[node as usize].key.clone(); self.unlink(node); + self.unlink_access(node); self.index.remove(&key); self.release(node); Some(key) @@ -3132,6 +3217,7 @@ impl CacheKeyOrder { let key = self.nodes[cursor as usize].key.clone(); self.index.remove(&key); self.unlink(cursor); + self.unlink_access(cursor); self.release(cursor); } cursor = next; @@ -3144,6 +3230,8 @@ impl CacheKeyOrder { self.index.clear(); self.head = CACHE_ORDER_NIL; self.tail = CACHE_ORDER_NIL; + self.access_head = CACHE_ORDER_NIL; + self.access_tail = CACHE_ORDER_NIL; } /// Iterate from most to least recently used. @@ -3162,18 +3250,30 @@ impl CacheKeyOrder { } } + /// Iterate the access order, least recently accessed first. + pub fn iter_access(&self) -> CacheKeyOrderAccessIter<'_> { + CacheKeyOrderAccessIter { + order: self, + cursor: self.access_head, + } + } + fn alloc(&mut self, key: CacheKey) -> u32 { if let Some(index) = self.free.pop() { let node = &mut self.nodes[index as usize]; node.key = key; node.prev = CACHE_ORDER_NIL; node.next = CACHE_ORDER_NIL; + node.access_prev = CACHE_ORDER_NIL; + node.access_next = CACHE_ORDER_NIL; return index; } self.nodes.push(CacheOrderNode { key, prev: CACHE_ORDER_NIL, next: CACHE_ORDER_NIL, + access_prev: CACHE_ORDER_NIL, + access_next: CACHE_ORDER_NIL, }); (self.nodes.len() - 1) as u32 } @@ -3206,6 +3306,47 @@ impl CacheKeyOrder { self.head = node; } + fn link_access_back(&mut self, node: u32) { + let old_tail = self.access_tail; + self.nodes[node as usize].access_prev = old_tail; + self.nodes[node as usize].access_next = CACHE_ORDER_NIL; + if old_tail == CACHE_ORDER_NIL { + self.access_head = node; + } else { + self.nodes[old_tail as usize].access_next = node; + } + self.access_tail = node; + } + + fn link_access_front(&mut self, node: u32) { + let old_head = self.access_head; + self.nodes[node as usize].access_next = old_head; + self.nodes[node as usize].access_prev = CACHE_ORDER_NIL; + if old_head == CACHE_ORDER_NIL { + self.access_tail = node; + } else { + self.nodes[old_head as usize].access_prev = node; + } + self.access_head = node; + } + + fn unlink_access(&mut self, node: u32) { + let prev = self.nodes[node as usize].access_prev; + let next = self.nodes[node as usize].access_next; + if prev == CACHE_ORDER_NIL { + self.access_head = next; + } else { + self.nodes[prev as usize].access_next = next; + } + if next == CACHE_ORDER_NIL { + self.access_tail = prev; + } else { + self.nodes[next as usize].access_prev = prev; + } + self.nodes[node as usize].access_prev = CACHE_ORDER_NIL; + self.nodes[node as usize].access_next = CACHE_ORDER_NIL; + } + fn unlink(&mut self, node: u32) { let prev = self.nodes[node as usize].prev; let next = self.nodes[node as usize].next; @@ -3244,6 +3385,24 @@ pub struct CacheKeyOrderRevIter<'a> { cursor: u32, } +pub struct CacheKeyOrderAccessIter<'a> { + order: &'a CacheKeyOrder, + cursor: u32, +} + +impl<'a> Iterator for CacheKeyOrderAccessIter<'a> { + type Item = &'a CacheKey; + + fn next(&mut self) -> Option { + if self.cursor == CACHE_ORDER_NIL { + return None; + } + let node = &self.order.nodes[self.cursor as usize]; + self.cursor = node.access_next; + Some(&node.key) + } +} + impl<'a> Iterator for CacheKeyOrderRevIter<'a> { type Item = &'a CacheKey; diff --git a/src/runtime/multilayer_cache.rs b/src/runtime/multilayer_cache.rs index 0c7e8e7..e1c85ea 100644 --- a/src/runtime/multilayer_cache.rs +++ b/src/runtime/multilayer_cache.rs @@ -71,21 +71,21 @@ pub trait CacheApi { fn capacity_cache(&self) -> usize; fn capacity_for_instance_cache(&self, instance_type: CacheInstanceType) -> usize { match instance_type { - CacheInstanceType::kUnified => self.capacity_cache(), + CacheInstanceType::Unified => self.capacity_cache(), _ => self.capacity_cache(), } } fn set_capacity_cache(&self, capacity: usize); fn set_capacity_for_instance_cache(&self, instance_type: CacheInstanceType, capacity: usize) { match instance_type { - CacheInstanceType::kUnified => self.set_capacity_cache(capacity), + CacheInstanceType::Unified => self.set_capacity_cache(capacity), _ => self.set_capacity_cache(capacity), } } fn size_cache(&self) -> usize; fn used_cache(&self, instance_type: CacheInstanceType) -> usize { match instance_type { - CacheInstanceType::kUnified => self.size_cache(), + CacheInstanceType::Unified => self.size_cache(), _ => self.size_cache(), } } @@ -180,28 +180,55 @@ struct SlotEvictionGroup { } impl SlotEvictionGroup { - fn new(victim: CacheKey, score: EvictionScore) -> Self { + fn new(victim: &CacheKey, score: EvictionScore) -> Self { Self { group_score: score, - victim, + victim: victim.clone(), victim_score: score, } } - fn observe(&mut self, key: CacheKey, score: EvictionScore) { + /// Fold one more member into the group. + /// + /// The key is borrowed and only cloned when it actually takes over as the + /// group's victim. Taking it by value cloned every key in the tier on + /// every eviction, and all but one of those clones was discarded. + fn observe(&mut self, key: &CacheKey, score: EvictionScore) { self.group_score.hotness = self.group_score.hotness.max(score.hotness); self.group_score.hits = self.group_score.hits.saturating_add(score.hits); self.group_score.last_access_epoch = self .group_score .last_access_epoch .max(score.last_access_epoch); - if score < self.victim_score || (score == self.victim_score && key < self.victim) { - self.victim = key; + if score < self.victim_score || (score == self.victim_score && *key < self.victim) { + self.victim = key.clone(); self.victim_score = score; } } } +/// How entries are grouped when picking an eviction victim. +/// +/// Entries that share a routing slot are weighed as a unit; everything else +/// stands alone. Borrowing the key's parts keeps this free of allocation: the +/// grouping used to render one `String` per resident entry per eviction, which +/// on a cache at capacity is a per-write cost proportional to the cache size. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum EvictionGroupKey<'a> { + Slot(u32), + Object(&'a str, &'a str), +} + +/// A chosen victim plus how many candidate groups were weighed to find it. +/// +/// The count feeds the `eviction_sampled_groups` statistic, which is what says +/// whether selection is inspecting a bounded number of candidates or the whole +/// tier. +struct PickedEvictionVictim { + victim: Option<(CacheKey, EvictionReason, u64)>, + groups_weighed: usize, +} + #[derive(Debug, Clone)] pub struct MultiLayerCache { inner: Arc>, @@ -232,12 +259,12 @@ struct CacheInner { memory: HashMap>, pmem: HashMap>, disk_index: HashMap, - disk_fifo_order: CacheKeyOrder, + disk_order: CacheKeyOrder, pinned: HashMap, pinned_handle_bytes: HashMap, pinned_removed_bytes: HashMap, - memory_fifo_order: CacheKeyOrder, - pmem_fifo_order: CacheKeyOrder, + memory_order: CacheKeyOrder, + pmem_order: CacheKeyOrder, async_writeback_queue: VecDeque, async_writeback_positions: HashMap, async_writeback_queue_bytes: u64, @@ -593,15 +620,15 @@ impl MultiLayerCache { ); cache.set_replacement_policy_for_tier( CacheTier::Memory, - CacheReplacementPolicy::from_reference_name(&options.cache_dram_replacement_policy), + CacheReplacementPolicy::from_config_name(&options.cache_dram_replacement_policy), ); cache.set_replacement_policy_for_tier( CacheTier::Pmem, - CacheReplacementPolicy::from_reference_name(&options.cache_pmem_replacement_policy), + CacheReplacementPolicy::from_config_name(&options.cache_pmem_replacement_policy), ); cache.set_replacement_policy_for_tier( CacheTier::Ssd, - CacheReplacementPolicy::from_reference_name(&options.cache_ssd_replacement_policy), + CacheReplacementPolicy::from_config_name(&options.cache_ssd_replacement_policy), ); cache.set_ssd_instance_only(options.cache_ssd_instance_only); cache.set_pmem_paths(options.pmem_paths); @@ -646,12 +673,12 @@ impl MultiLayerCache { memory: HashMap::new(), pmem: HashMap::new(), disk_index: HashMap::new(), - disk_fifo_order: CacheKeyOrder::new(), + disk_order: CacheKeyOrder::new(), pinned: HashMap::new(), pinned_handle_bytes: HashMap::new(), pinned_removed_bytes: HashMap::new(), - memory_fifo_order: CacheKeyOrder::new(), - pmem_fifo_order: CacheKeyOrder::new(), + memory_order: CacheKeyOrder::new(), + pmem_order: CacheKeyOrder::new(), async_writeback_queue: VecDeque::new(), async_writeback_positions: HashMap::new(), async_writeback_queue_bytes: 0, @@ -721,12 +748,27 @@ impl MultiLayerCache { self.inner.read().expect("cache lock poisoned").started } + /// Total capacity across the tiers, counted the way `size` counts usage. + /// + /// Side-by-side placement holds distinct keys in Dram and Pmem, so the two + /// capacities add. Tiered placement holds a given key in at most one of + /// them, so the pair contributes the larger of the two rather than their + /// sum. Summing under tiered placement overstates capacity, and since + /// `size` already takes the maximum there, it made a full cache report as + /// roughly half used. pub fn capacity(&self) -> usize { let inner = self.inner.read().expect("cache lock poisoned"); - inner - .memory_capacity_bytes - .saturating_add(inner.pmem_capacity_bytes) - .max(inner.ssd_capacity_bytes) + let volatile_bytes = if matches!( + inner.tiering_policy.data_placement, + CacheDataPlacement::SideBySide + ) { + inner + .memory_capacity_bytes + .saturating_add(inner.pmem_capacity_bytes) + } else { + inner.memory_capacity_bytes.max(inner.pmem_capacity_bytes) + }; + volatile_bytes.max(inner.ssd_capacity_bytes) } pub fn capacity_for_tier(&self, tier: CacheTier) -> usize { @@ -1218,8 +1260,10 @@ impl MultiLayerCache { if let Some(value) = inner.memory.get(key).cloned() { inner.stats.memory_hits += 1; inner.record_hit(key, value.len()); - inner.record_get_latency(started); - inner.record_read_through_latency(started); + // One interval, two histograms: read the clock once. + let micros = elapsed_micros(started); + inner.record_get_latency_micros(micros); + inner.record_read_through_latency_micros(micros); return Ok(Some(CacheReadResult { value: value.to_vec(), tier: CacheReadTier::Memory, @@ -2477,7 +2521,7 @@ impl MultiLayerCache { #[allow(non_snake_case)] pub fn SetDRAMPMEMDataPlacementType(&self, placement: DRAMPMEMDataPlacementType) { - self.set_reference_data_placement_type(placement); + self.set_config_data_placement_type(placement); } #[allow(non_snake_case)] @@ -2487,7 +2531,7 @@ impl MultiLayerCache { #[allow(non_snake_case)] pub fn GetDRAMPMEMDataPlacementType(&self) -> DRAMPMEMDataPlacementType { - self.reference_data_placement_type() + self.config_data_placement_type() } #[allow(non_snake_case)] @@ -2709,7 +2753,7 @@ impl MultiLayerCache { .data_placement } - pub fn reference_data_placement_type(&self) -> DRAMPMEMDataPlacementType { + pub fn config_data_placement_type(&self) -> DRAMPMEMDataPlacementType { self.data_placement().into() } @@ -2718,7 +2762,7 @@ impl MultiLayerCache { inner.tiering_policy.data_placement = placement; } - pub fn set_reference_data_placement_type(&self, placement: DRAMPMEMDataPlacementType) { + pub fn set_config_data_placement_type(&self, placement: DRAMPMEMDataPlacementType) { self.set_data_placement(placement.into()); } @@ -2748,8 +2792,8 @@ impl MultiLayerCache { if enabled { inner.memory.clear(); inner.pmem.clear(); - inner.memory_fifo_order.clear(); - inner.pmem_fifo_order.clear(); + inner.memory_order.clear(); + inner.pmem_order.clear(); inner.memory_bytes = 0; inner.pmem_bytes = 0; inner.refresh_usage_stats(); @@ -4816,7 +4860,7 @@ impl CacheInner { if let Some(old_len) = self.disk_index.insert(key.clone(), block_len as u64) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len as u64); self.record_metadata( &key, @@ -5028,10 +5072,10 @@ impl CacheInner { .collect::>(); if staged_keys.len() > SET_MEMBERSHIP_THRESHOLD { let staged_key_set = staged_keys.iter().cloned().collect::>(); - self.disk_fifo_order + self.disk_order .retain(|candidate| !staged_key_set.contains(candidate)); } else { - self.disk_fifo_order + self.disk_order .retain(|candidate| !staged_keys.contains(candidate)); } } @@ -5039,7 +5083,7 @@ impl CacheInner { if let Some(old_len) = self.disk_index.insert(entry.key.clone(), entry.block_len) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } - self.disk_fifo_order.push_back_if_absent(entry.key.clone()); + self.disk_order.push_back_if_absent(entry.key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(entry.block_len); self.record_metadata( &entry.key, @@ -5116,7 +5160,7 @@ impl CacheInner { if let Some(old_len) = self.disk_index.insert(key.clone(), block_len) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len); self.record_metadata( &key, @@ -5178,7 +5222,7 @@ impl CacheInner { if let Some(old_len) = self.disk_index.insert(key.clone(), block_len as u64) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len as u64); self.record_metadata( &key, @@ -5208,7 +5252,7 @@ impl CacheInner { self.pinned_removed_bytes.insert(key.clone(), value.len()); } } - self.memory_fifo_order.remove(key); + self.memory_order.remove(key); } CacheTier::Pmem => { if let Some(value) = self.pmem.remove(key) { @@ -5217,7 +5261,7 @@ impl CacheInner { self.pinned_removed_bytes.insert(key.clone(), value.len()); } } - self.pmem_fifo_order.remove(key); + self.pmem_order.remove(key); self.persist_pmem_delete(key)?; } CacheTier::Ssd => { @@ -5225,7 +5269,7 @@ impl CacheInner { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } self.delete_ssd_block(key)?; - self.disk_fifo_order.remove(key); + self.disk_order.remove(key); self.stats.disk_bytes = self.ssd_bytes; self.append_disk_manifest_delete(key)?; } @@ -5326,20 +5370,20 @@ impl CacheInner { .remove(key) .unwrap_or(existing_block.len() as u64); self.ssd_bytes = self.ssd_bytes.saturating_sub(indexed_old_len); - self.disk_fifo_order.remove(key); + self.disk_order.remove(key); self.evict_ssd_for(block.len() as u64); if self.ssd_bytes.saturating_add(block.len() as u64) > self.ssd_capacity_bytes as u64 { self.disk_index.insert(key.clone(), indexed_old_len); - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(indexed_old_len); self.stats.disk_bytes = self.ssd_bytes; return Err(CacheError::CapacityExceeded); } self.write_ssd_block(key, &block)?; self.disk_index.insert(key.clone(), block.len() as u64); - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block.len() as u64); self.record_metadata( key, @@ -5410,9 +5454,9 @@ impl MultiLayerCache { } } inner - .memory_fifo_order + .memory_order .retain(|key| key.shard_id != shard_id); - inner.pmem_fifo_order.retain(|key| key.shard_id != shard_id); + inner.pmem_order.retain(|key| key.shard_id != shard_id); let disk_keys = inner .disk_index .keys() @@ -5425,7 +5469,7 @@ impl MultiLayerCache { disk_bytes_before.saturating_add(inner.disk_index.remove(key).unwrap_or_default()); } let _ = inner.delete_ssd_blocks(&disk_keys); - inner.disk_fifo_order.retain(|key| key.shard_id != shard_id); + inner.disk_order.retain(|key| key.shard_id != shard_id); inner.metadata.retain(|key, _| key.shard_id != shard_id); inner.pinned.retain(|key, _| key.shard_id != shard_id); inner @@ -5492,13 +5536,13 @@ impl MultiLayerCache { } let _ = inner.delete_ssd_blocks(&disk_delete_keys); inner - .memory_fifo_order + .memory_order .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); inner - .pmem_fifo_order + .pmem_order .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); inner - .disk_fifo_order + .disk_order .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); inner.stats.invalidations = inner .stats @@ -5559,13 +5603,13 @@ impl MultiLayerCache { inner.metadata.remove(key); } let _ = inner.delete_ssd_blocks(&disk_delete_keys); - inner.memory_fifo_order.retain(|key| { + inner.memory_order.retain(|key| { !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) }); - inner.pmem_fifo_order.retain(|key| { + inner.pmem_order.retain(|key| { !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) }); - inner.disk_fifo_order.retain(|key| { + inner.disk_order.retain(|key| { !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) }); inner.stats.invalidations = inner diff --git a/src/runtime/replacement.rs b/src/runtime/replacement.rs index 20d6d56..2db82a9 100644 --- a/src/runtime/replacement.rs +++ b/src/runtime/replacement.rs @@ -240,7 +240,7 @@ pub enum CacheReplacementPolicy { } impl CacheReplacementPolicy { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("fifo") { Self::Fifo } else if value.eq_ignore_ascii_case("slru") { @@ -250,7 +250,7 @@ impl CacheReplacementPolicy { } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { Self::Fifo => "FIFO", Self::Slru => "SLRU", @@ -1028,6 +1028,13 @@ impl ReplacementArc { Ok(()) } + /// Drop every tracked key and put this policy back in the uninitialized + /// state. + /// + /// Unlike the Fifo and SLRU policies, resetting this one *does* require + /// another `init()` before it will accept keys. That asymmetry is + /// intentional and load-bearing for callers that drive the adaptive + /// lists directly; do not "harmonize" it away. pub fn reset(&mut self) -> Result<(), CacheError> { self.initialized = false; self.arc_list.reset(); @@ -1243,12 +1250,18 @@ impl ReplacementFIFO { self.base.init() } + /// Drop every tracked buffer and return the policy to an empty state. + /// + /// A successful reset leaves the policy *initialized and usable*: it + /// empties the index, not the policy's lifecycle. Clearing the + /// initialized flag here would make every later `put` return "no + /// evictions" while silently discarding the buffer, turning a reset + /// cache into a black hole that never reports an error. pub fn reset(&mut self) -> Result<(), CacheError> { self.index.clear(); self.queue.clear(); self.arena.clear(); self.base.used = 0; - self.base.initialized = false; Ok(()) } @@ -1485,7 +1498,7 @@ struct SlruEntry { node: u32, } -/// A hash-partitioned shard: three LRU lists plus the shard byte accounting. +/// A hash-partitioned shard: three Lru lists plus the shard byte accounting. #[derive(Debug)] struct SlruSegment { lists: [KeyList; SLRU_LIST_COUNT], @@ -1522,10 +1535,10 @@ enum SlruMaintainerStep { Evicted(CacheBuffer), } -/// Segmented LRU replacement policy. +/// Segmented Lru replacement policy. /// /// The cache is hash-partitioned into [`ReplacementSLRU::num_segments`] shards, -/// each holding an independent hot/warm/cold LRU triple with its own byte +/// each holding an independent hot/warm/cold Lru triple with its own byte /// budget of `capacity / num_segments`. Inserts, lookups and eviction only ever /// touch the shard owning the key, so eviction scans a small shard-local list /// instead of one global list, and no single hot list can consume the whole @@ -1585,6 +1598,11 @@ impl ReplacementSLRU { self.base.init() } + /// Drop every tracked buffer and return the policy to an empty state. + /// + /// As with the other policies, a successful reset leaves this one + /// initialized and ready to accept buffers again; see the note on + /// `ReplacementFIFO::reset`. pub fn reset(&mut self) -> Result<(), CacheError> { self.index.clear(); for segment in &mut self.segments { @@ -1595,7 +1613,6 @@ impl ReplacementSLRU { } self.arena.clear(); self.base.used = 0; - self.base.initialized = false; Ok(()) } @@ -1665,7 +1682,7 @@ impl ReplacementSLRU { /// /// Only the access flag moves: an entry is not re-fronted in its list on a /// read. List position is decided by the maintainer and by eviction, which - /// is what keeps this a segmented policy rather than a plain LRU — reading + /// is what keeps this a segmented policy rather than a plain Lru — reading /// a key once does not jump it ahead of one that was read twice. pub fn get(&mut self, key: &str) -> Option { let entry = self.index.get_mut(key)?; @@ -2243,7 +2260,7 @@ impl ReplacementSLRU { } } -/// A segmented LRU that can be shared across threads, holding one lock per +/// A segmented Lru that can be shared across threads, holding one lock per /// segment instead of one lock over the whole policy. /// /// [`ReplacementSLRU`] partitions its lists into segments, but it is driven diff --git a/src/runtime/storage_engines.rs b/src/runtime/storage_engines.rs index 3b8a627..9f6b9d2 100644 --- a/src/runtime/storage_engines.rs +++ b/src/runtime/storage_engines.rs @@ -81,7 +81,7 @@ impl CacheOptions { } pub fn with_replacement_policy(mut self, policy: CacheReplacementPolicy) -> Self { - let name = policy.as_reference_name().to_string(); + let name = policy.as_config_name().to_string(); self.cache_dram_replacement_policy = name.clone(); self.cache_pmem_replacement_policy = name.clone(); self.cache_ssd_replacement_policy = name; @@ -93,7 +93,7 @@ impl CacheOptions { tier: CacheTier, policy: CacheReplacementPolicy, ) -> Self { - let name = policy.as_reference_name().to_string(); + let name = policy.as_config_name().to_string(); match tier { CacheTier::Memory => self.cache_dram_replacement_policy = name, CacheTier::Pmem => self.cache_pmem_replacement_policy = name, @@ -108,12 +108,12 @@ impl CacheOptions { placement: CacheDataPlacement, threshold: usize, ) -> Self { - self.cache_dram_pmem_data_placement_type = placement.as_reference_name().to_string(); + self.cache_dram_pmem_data_placement_type = placement.as_config_name().to_string(); self.cache_dram_pmem_data_placement_threshold = threshold; self } - pub fn with_reference_dram_pmem_data_placement( + pub fn with_config_dram_pmem_data_placement( self, placement: DRAMPMEMDataPlacementType, threshold: usize, @@ -127,7 +127,7 @@ impl CacheOptions { placement: DRAMPMEMDataPlacementType, threshold: usize, ) -> Self { - self.with_reference_dram_pmem_data_placement(placement, threshold) + self.with_config_dram_pmem_data_placement(placement, threshold) } pub fn with_metric_id_prefix(mut self, prefix: impl Into) -> Self { @@ -171,7 +171,7 @@ impl CacheOptions { memory_capacity_bytes: self.dram_capacity, pmem_capacity_bytes: self.pmem_capacity, ssd_capacity_bytes: self.ssd_capacity, - data_placement: CacheDataPlacement::from_reference_name( + data_placement: CacheDataPlacement::from_config_name( &self.cache_dram_pmem_data_placement_type, ), data_placement_threshold_bytes: self.cache_dram_pmem_data_placement_threshold, @@ -1401,7 +1401,7 @@ impl PmemAllocatorRecoverListenerImpl { continue; }; let buffer = - MemStorage::create_cache_buffer_from_data(record, StorageEngineType::kPMEM, true)?; + MemStorage::create_cache_buffer_from_data(record, StorageEngineType::Pmem, true)?; callback.on_recover_data(&key, buffer); valid_records = valid_records.saturating_add(1); } @@ -1464,7 +1464,7 @@ impl StorageEngineSimple { } fn buffer_from_record(&self, record: &[u8]) -> Result { - MemStorage::create_cache_buffer_from_data(record, StorageEngineType::kSimple, false) + MemStorage::create_cache_buffer_from_data(record, StorageEngineType::Simple, false) } pub fn test_get_num_delete_completed_count(&self) -> u32 { @@ -1634,7 +1634,7 @@ impl StorageEngineApi for StorageEngineSimple { } fn storage_engine_type(&self) -> StorageEngineType { - StorageEngineType::kSimple + StorageEngineType::Simple } } @@ -2389,7 +2389,7 @@ impl StorageEngineApi for StorageEngineRocksDB { } fn storage_engine_type(&self) -> StorageEngineType { - StorageEngineType::kSSD + StorageEngineType::Ssd } } @@ -2475,7 +2475,7 @@ pub struct StorageEngineMultiSSD { impl StorageEngineMultiSSD { pub fn new(paths: impl IntoIterator, capacity: u64) -> Self { - Self::with_type(paths, capacity, StorageEngineType::kSSD) + Self::with_type(paths, capacity, StorageEngineType::Ssd) } pub fn with_paths(paths: impl IntoIterator, capacity: u64) -> Self { @@ -2757,7 +2757,7 @@ impl StorageEngineApi for StorageEngineMultiSSD { } fn storage_engine_type(&self) -> StorageEngineType { - StorageEngineType::kMultiSSD + StorageEngineType::MultiSsd } } @@ -3036,7 +3036,7 @@ impl PMemDispatcher { }) .collect(); Self { - alloc_type: AllocatorType::kLogBasedAllocator, + alloc_type: AllocatorType::LogBasedAllocator, writers, current_numa: 0, stopped: true, @@ -3046,7 +3046,7 @@ impl PMemDispatcher { pub fn from_allocators(allocators: Vec) -> Self { let writers = allocators.into_iter().map(AsyncWriter::new).collect(); Self { - alloc_type: AllocatorType::kLogBasedAllocator, + alloc_type: AllocatorType::LogBasedAllocator, writers, current_numa: 0, stopped: true, diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 638284e..ad9ca87 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -6,7 +6,7 @@ mod tests { use super::*; #[test] - fn parity_rdma_response_hash_table_and_index_surface_round_trip() { + fn rdma_response_hash_table_and_index_surface_round_trip() { let mut response = RDMAResponse::New(32); let first_allocation = response.allocation_addr(); assert_eq!(response.GetRespSize(), 32); @@ -44,9 +44,9 @@ mod tests { entry.set_signature_96(sig96); entry.SetDataLength(128); entry.SetVersion(); - entry.set_packed_addr(0x1234, RdmaStorageEngineType::PMEM, 128); + entry.set_packed_addr(0x1234, RdmaStorageEngineType::Pmem, 128); assert_eq!(entry.GetPtr(), 0x1234); - assert_eq!(entry.GetType(), RdmaStorageEngineType::PMEM.as_code()); + assert_eq!(entry.GetType(), RdmaStorageEngineType::Pmem.as_code()); assert_eq!(entry.GetOverflowFlag(), 0); assert_eq!(entry.GetLength(), 128); assert_eq!(entry.GetVersion(), 0); @@ -56,9 +56,9 @@ mod tests { overflow.set_signature_128(sig128); overflow.SetDataLength(i32::MAX); overflow.SetVersion(); - overflow.set_packed_addr(0x2222, RdmaStorageEngineType::SSD, RDMA_MAX_BLOCK_SIZE + 1); + overflow.set_packed_addr(0x2222, RdmaStorageEngineType::Ssd, RDMA_MAX_BLOCK_SIZE + 1); assert_eq!(overflow.GetPtr(), 0x2222); - assert_eq!(overflow.GetType(), RdmaStorageEngineType::SSD.as_code()); + assert_eq!(overflow.GetType(), RdmaStorageEngineType::Ssd.as_code()); assert_eq!(overflow.GetOverflowFlag(), 1); assert_eq!(overflow.GetSignature128b(), sig128); @@ -67,7 +67,7 @@ mod tests { assert_eq!(table.GetNumEntries(), 0); assert!(table.AllBucketsUnlocked()); - let put = table.Put(key.clone(), 0x1000, 11, RdmaStorageEngineType::DRAM); + let put = table.Put(key.clone(), 0x1000, 11, RdmaStorageEngineType::Dram); assert_eq!(put.status, RDMA_OP_SUCCESS); assert_eq!(put.old_addr, None); assert_eq!(table.GetNumEntries(), 1); @@ -75,31 +75,31 @@ mod tests { let got = table.Get(&key); assert_eq!(got.addr, Some(0x1000)); assert_eq!(got.len, 11 + RDMA_DATA_HEADER + RDMA_CRC_LEN); - assert_eq!(got.storage_type, RdmaStorageEngineType::DRAM); + assert_eq!(got.storage_type, RdmaStorageEngineType::Dram); - let update = table.Put(key.clone(), 0x2000, 17, RdmaStorageEngineType::SSD); + let update = table.Put(key.clone(), 0x2000, 17, RdmaStorageEngineType::Ssd); assert_eq!(update.status, RDMA_OP_SUCCESS); assert_eq!(update.old_addr, Some(0x1000)); assert_eq!(update.old_len, 11 + RDMA_DATA_HEADER + RDMA_CRC_LEN); - assert_eq!(update.old_type, RdmaStorageEngineType::DRAM); + assert_eq!(update.old_type, RdmaStorageEngineType::Dram); assert_eq!(table.GetNumEntries(), 1); let got = table.Get(&key); assert_eq!(got.addr, Some(0x2000)); - assert_eq!(got.storage_type, RdmaStorageEngineType::SSD); + assert_eq!(got.storage_type, RdmaStorageEngineType::Ssd); let del = table.Del(&key); assert_eq!(del.status, RDMA_OP_SUCCESS); assert_eq!(del.addr, Some(0x2000)); - assert_eq!(del.storage_type, RdmaStorageEngineType::SSD); + assert_eq!(del.storage_type, RdmaStorageEngineType::Ssd); assert_eq!(table.GetNumEntries(), 0); - assert_eq!(table.Get(&key).storage_type, RdmaStorageEngineType::INVALID); + assert_eq!(table.Get(&key).storage_type, RdmaStorageEngineType::Invalid); assert_eq!(table.Del(&key).status, RDMA_NOT_FOUND); assert!(table.AllBucketsUnlocked()); } #[test] - fn parity_rdma_std_allocator_allocates_and_frees_virtual_regions() { + fn rdma_std_allocator_allocates_and_frees_virtual_regions() { fn round_trip(allocator: &mut A) -> AllocatorPtr { let addr = allocator.allocate(64).expect("allocator ptr"); allocator.free(addr, 64); @@ -123,7 +123,7 @@ mod tests { } #[test] - fn parity_rdma_dram_and_pmem_storage_engines_round_trip_blocks() { + fn rdma_dram_and_pmem_storage_engines_round_trip_blocks() { let mut dram = RdmaStorageEngineDram::with_capacity(1024); let key = 1_i32.to_le_bytes(); let value = 7_i32.to_le_bytes(); @@ -173,25 +173,25 @@ mod tests { } #[test] - fn parity_rdma_cache_composes_index_storage_and_replacement_policy() { + fn rdma_cache_composes_index_storage_and_replacement_policy() { let key = 1_i32.to_le_bytes(); let value_one = 1_i32.to_le_bytes(); let value_two = 2_i32.to_le_bytes(); - let mut cache = RDMACache::new(1024, 1024, 1024, RdmaReplacementPolicyType::FIFO); - assert_eq!(cache.GetCapacity(RdmaStorageEngineType::DRAM), 1024); + let mut cache = RDMACache::new(1024, 1024, 1024, RdmaReplacementPolicyType::Fifo); + assert_eq!(cache.GetCapacity(RdmaStorageEngineType::Dram), 1024); assert_eq!( cache.GetReplacementPolicyType(), - RdmaReplacementPolicyType::FIFO + RdmaReplacementPolicyType::Fifo ); - cache.SetReplacementPolicy(RdmaReplacementPolicyType::LRU); + cache.SetReplacementPolicy(RdmaReplacementPolicyType::Lru); assert_eq!( cache.GetReplacementPolicyType(), - RdmaReplacementPolicyType::LRU + RdmaReplacementPolicyType::Lru ); assert_eq!( - RdmaReplacementPolicyType::LRU.as_replacement_policy_type(), - ReplacementPolicyType::kLRU + RdmaReplacementPolicyType::Lru.as_replacement_policy_type(), + ReplacementPolicyType::Lru ); let mut response = RDMAResponse::new(); @@ -200,7 +200,7 @@ mod tests { assert_eq!(response.GetResponse(), value_one); assert_eq!(cache.num_index_entries(), 1); assert_eq!( - cache.storage_stats(RdmaStorageEngineType::DRAM).unwrap().2, + cache.storage_stats(RdmaStorageEngineType::Dram).unwrap().2, 1 ); @@ -210,7 +210,7 @@ mod tests { assert_eq!(response.GetResponse(), value_two); assert_eq!(cache.num_index_entries(), 1); assert_eq!( - cache.storage_stats(RdmaStorageEngineType::DRAM).unwrap().2, + cache.storage_stats(RdmaStorageEngineType::Dram).unwrap().2, 1 ); @@ -222,7 +222,7 @@ mod tests { let pmem_key = b"pmem-key"; assert_eq!( - cache.InsertToStorage(RdmaStorageEngineType::PMEM, pmem_key, b"pmem-value"), + cache.InsertToStorage(RdmaStorageEngineType::Pmem, pmem_key, b"pmem-value"), RDMA_OP_SUCCESS ); response.Clear(); @@ -231,7 +231,7 @@ mod tests { let ssd_key = b"ssd-key"; assert_eq!( - cache.InsertToStorage(RdmaStorageEngineType::SSD, ssd_key, b"ssd-value"), + cache.InsertToStorage(RdmaStorageEngineType::Ssd, ssd_key, b"ssd-value"), RDMA_OP_SUCCESS ); response.Clear(); @@ -241,12 +241,12 @@ mod tests { let mut dram_only = RDMACache::with_dram_capacity(8); assert_eq!(dram_only.Insert(b"too-large", b"value"), RDMA_FAIL_ALLOC); assert_eq!( - dram_only.InsertToStorage(RdmaStorageEngineType::PMEM, b"k", b"v"), + dram_only.InsertToStorage(RdmaStorageEngineType::Pmem, b"k", b"v"), RDMA_FAIL_ALLOC ); - dram_only.InitStorageEngine(RdmaStorageEngineType::PMEM, 128); + dram_only.InitStorageEngine(RdmaStorageEngineType::Pmem, 128); assert_eq!( - dram_only.InsertToStorage(RdmaStorageEngineType::PMEM, b"k", b"v"), + dram_only.InsertToStorage(RdmaStorageEngineType::Pmem, b"k", b"v"), RDMA_OP_SUCCESS ); } @@ -301,7 +301,40 @@ mod tests { } #[test] - fn parity_unified_size_is_placement_aware() { + fn unified_capacity_is_placement_aware() { + let dir = tempfile::tempdir().unwrap(); + let cache = MultiLayerCache::with_tiering_policy( + dir.path(), + CacheTieringPolicy { + memory_capacity_bytes: 64, + pmem_capacity_bytes: 64, + ssd_capacity_bytes: 16, + data_placement: CacheDataPlacement::Tiered, + data_placement_threshold_bytes: 4, + memory_hotness_threshold: 0, + pmem_admit_hotness_threshold: 0, + ssd_admit_hotness_threshold: u32::MAX, + max_memory_block_bytes: 64, + max_pmem_block_bytes: 64, + max_ssd_block_bytes: 16, + ssd_write_through: false, + }, + CacheBlockOptions::default(), + ); + + // Tiered placement holds a key in at most one of the volatile tiers, + // so the pair contributes the larger of the two. This is the same rule + // Size already applies, and summing here would report a full cache as + // half used. + assert_eq!(cache.Capacity(), 64); + + // Side by side holds distinct keys in each tier, so they add. + cache.SetDataPlacementType(CacheDataPlacement::SideBySide); + assert_eq!(cache.Capacity(), 128); + } + + #[test] + fn unified_size_is_placement_aware() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -324,11 +357,11 @@ mod tests { let memory_key = CacheKey::string(7, "memory-size"); let pmem_key = CacheKey::string(7, "pmem-size"); cache - .TEST_Insert(CacheInstanceType::kDRAM, memory_key, b"abcd".to_vec(), 4) + .TEST_Insert(CacheInstanceType::Dram, memory_key, b"abcd".to_vec(), 4) .unwrap(); cache .TEST_Insert( - CacheInstanceType::kPMEM, + CacheInstanceType::Pmem, pmem_key, b"0123456789".to_vec(), 10, @@ -344,7 +377,7 @@ mod tests { } #[test] - fn parity_cache_api_aliases_match_insert_lookup_remove_semantics() { + fn cache_api_aliases_match_insert_lookup_remove_semantics() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(32, dir.path()); let key = CacheKey::string(31, "legacy-api"); @@ -791,7 +824,7 @@ mod tests { } #[test] - fn parity_simple_lru_cache_wrapper_evicts_like_public_stub_cache() { + fn simple_lru_cache_wrapper_evicts_like_public_stub_cache() { let cache = MatrixCacheBuilder::BuildSimpleLRUCache(12); assert!(cache.Stop()); assert!(cache.Start()); @@ -837,7 +870,7 @@ mod tests { } #[test] - fn parity_zero_copy_simple_lru_cache_keeps_removed_pinned_value_readable() { + fn zero_copy_simple_lru_cache_keeps_removed_pinned_value_readable() { let cache = MatrixCacheBuilder::BuildZeroCopySimpleLRUCache(8); let pinned_key = CacheKey::string(33, "pinned"); let cold_key = CacheKey::string(33, "cold"); @@ -899,7 +932,7 @@ mod tests { } #[test] - fn parity_string_cache_wrappers_match_tool_cache_interface() { + fn string_cache_wrappers_match_tool_cache_interface() { let simple = MatrixCacheBuilder::BuildConcurrentSimpleLRUCache(16); assert!(simple.Stop()); assert!(simple.Start()); @@ -930,16 +963,16 @@ mod tests { simple.RemoveAll().unwrap(); assert_eq!(simple.Size(), 0); - let exact_reference_name = ConcurrentSimpleLRUCache::new(32); - exact_reference_name + let exact_config_name = ConcurrentSimpleLRUCache::new(32); + exact_config_name .InsertDefaultSize("gamma", "three".to_string()) .unwrap(); assert_eq!( - exact_reference_name.Lookup("gamma").unwrap(), + exact_config_name.Lookup("gamma").unwrap(), Some("three".to_string()) ); - let string_api: &dyn StringCacheApi = &exact_reference_name; + let string_api: &dyn StringCacheApi = &exact_config_name; string_api .insert_string_default_size("delta", "four".to_string()) .unwrap(); @@ -950,7 +983,7 @@ mod tests { } #[test] - fn parity_memcached_wrapper_matches_tool_cache_surface_without_external_daemon() { + fn memcached_wrapper_matches_tool_cache_surface_without_external_daemon() { let cache = MatrixCacheBuilder::BuildMemcachedWrapper(8); assert_eq!(cache.configured_capacity(), 8); assert_eq!(cache.Capacity(), 8); @@ -1001,7 +1034,7 @@ mod tests { } #[test] - fn parity_multi_tier_string_cache_wraps_zero_copy_cache() { + fn multi_tier_string_cache_wraps_zero_copy_cache() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildMultiTierStringCache(CacheOptions { dram_capacity: 8, @@ -1034,7 +1067,7 @@ mod tests { } #[test] - fn parity_pascal_case_cache_methods_match_matrixcache_interface() { + fn pascal_case_cache_methods_match_matrixcache_interface() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache(CacheOptions { dram_capacity: 64, @@ -1109,7 +1142,7 @@ mod tests { } #[test] - fn parity_instance_controls_match_unified_cache_getters_and_setters() { + fn instance_controls_match_unified_cache_getters_and_setters() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -1130,26 +1163,26 @@ mod tests { CacheBlockOptions::default(), ); - assert_eq!(cache.GetCapacity(CacheInstanceType::kDRAM), 16); - assert_eq!(cache.GetCapacity(CacheInstanceType::kPMEM), 32); - assert_eq!(cache.GetCapacity(CacheInstanceType::kSSD), 128); - assert_eq!(cache.GetCapacity(CacheInstanceType::kUnified), 128); + assert_eq!(cache.GetCapacity(CacheInstanceType::Dram), 16); + assert_eq!(cache.GetCapacity(CacheInstanceType::Pmem), 32); + assert_eq!(cache.GetCapacity(CacheInstanceType::Ssd), 128); + assert_eq!(cache.GetCapacity(CacheInstanceType::Unified), 128); - cache.SetCapacityForInstance(CacheInstanceType::kDRAM, 8); - cache.SetCapacityForInstance(CacheInstanceType::kPMEM, 24); - cache.SetCapacityForInstance(CacheInstanceType::kSSD, 96); - assert_eq!(cache.get_capacity(CacheInstanceType::kDRAM), 8); - assert_eq!(cache.get_capacity(CacheInstanceType::kPMEM), 24); - assert_eq!(cache.get_capacity(CacheInstanceType::kSSD), 96); + cache.SetCapacityForInstance(CacheInstanceType::Dram, 8); + cache.SetCapacityForInstance(CacheInstanceType::Pmem, 24); + cache.SetCapacityForInstance(CacheInstanceType::Ssd, 96); + assert_eq!(cache.get_capacity(CacheInstanceType::Dram), 8); + assert_eq!(cache.get_capacity(CacheInstanceType::Pmem), 24); + assert_eq!(cache.get_capacity(CacheInstanceType::Ssd), 96); - cache.SetReplacementPolicyType(CacheInstanceType::kDRAM, CacheReplacementPolicy::Fifo); - cache.SetReplacementPolicyType(CacheInstanceType::kPMEM, CacheReplacementPolicy::Slru); + cache.SetReplacementPolicyType(CacheInstanceType::Dram, CacheReplacementPolicy::Fifo); + cache.SetReplacementPolicyType(CacheInstanceType::Pmem, CacheReplacementPolicy::Slru); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::Fifo ); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kPMEM), + cache.GetReplacementPolicyType(CacheInstanceType::Pmem), CacheReplacementPolicy::Slru ); @@ -1162,31 +1195,31 @@ mod tests { cache .Insert(memory_key.clone(), b"abcd".to_vec(), b"abcd".len()) .unwrap(); - assert!(cache.GetUsed(CacheInstanceType::kDRAM) > 0); + assert!(cache.GetUsed(CacheInstanceType::Dram) > 0); assert!(cache.Size() >= b"abcd".len()); } #[test] - fn parity_allocator_types_and_stats_match_cache_instance_storage_surface() { + fn allocator_types_and_stats_match_cache_instance_storage_surface() { assert_eq!( - AllocatorType::from_reference_name("kLogBasedAllocator"), - AllocatorType::kLogBasedAllocator + AllocatorType::from_config_name("kLogBasedAllocator"), + AllocatorType::LogBasedAllocator ); assert_eq!( - AllocatorType::from_reference_name("pool_based"), - AllocatorType::kPoolBasedAllocator + AllocatorType::from_config_name("pool_based"), + AllocatorType::PoolBasedAllocator ); - assert_eq!(AllocatorType::kJeAllocator.as_reference_name(), "JeAllocator"); + assert_eq!(AllocatorType::JeAllocator.as_config_name(), "JeAllocator"); let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); assert_eq!( instance.GetAllocatorType(), - AllocatorType::kPoolBasedAllocator + AllocatorType::PoolBasedAllocator ); assert_eq!(instance.GetAllocatorStats(), AllocatorStats::default()); @@ -1206,11 +1239,11 @@ mod tests { } #[test] - fn parity_cache_instance_latency_summary_uses_live_cache_metrics() { + fn cache_instance_latency_summary_uses_live_cache_metrics() { let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); instance.Put("latency-a", b"abc".to_vec()).unwrap(); @@ -1230,7 +1263,7 @@ mod tests { } #[test] - fn parity_allocator_metadata_structs_preserve_chunk_state() { + fn allocator_metadata_structs_preserve_chunk_state() { let stats = AllocatorStats::new(128, 32); assert_eq!(stats.NumAllocatedBytes(), 128); assert_eq!(stats.NumFreedBytes(), 32); @@ -1255,20 +1288,20 @@ mod tests { } #[test] - fn parity_allocator_recovery_surface_matches_pmem_and_pool_headers() { - assert_eq!(AllocatorType::kLogBasedAllocator as u8, 0); - assert_eq!(AllocatorType::kPoolBasedAllocator as u8, 1); - assert_eq!(AllocatorType::kJeAllocator as u8, 2); - assert_eq!(AllocatorType::kMaxCode as u8, 3); + fn allocator_recovery_surface_matches_pmem_and_pool_headers() { + assert_eq!(AllocatorType::LogBasedAllocator as u8, 0); + assert_eq!(AllocatorType::PoolBasedAllocator as u8, 1); + assert_eq!(AllocatorType::JeAllocator as u8, 2); + assert_eq!(AllocatorType::MaxCode as u8, 3); - assert_eq!(FlushPolicy::kNoFlush as u8, 0); - assert_eq!(FlushPolicy::kInstantFlush as u8, 1); - assert_eq!(FlushPolicy::kMiniBatchFlush as u8, 2); + assert_eq!(FlushPolicy::NoFlush as u8, 0); + assert_eq!(FlushPolicy::InstantFlush as u8, 1); + assert_eq!(FlushPolicy::MiniBatchFlush as u8, 2); assert_eq!( - FlushPolicy::from_reference_name("kInstantFlush"), - FlushPolicy::kInstantFlush + FlushPolicy::from_config_name("kInstantFlush"), + FlushPolicy::InstantFlush ); - assert_eq!(FlushPolicy::kMiniBatchFlush.as_reference_name(), "MiniBatchFlush"); + assert_eq!(FlushPolicy::MiniBatchFlush.as_config_name(), "MiniBatchFlush"); let mut recover = PmemRecoverStats::default(); recover.AddChunkStats(ChunkRecoverStats { @@ -1312,7 +1345,7 @@ mod tests { } #[test] - fn parity_specialized_allocator_aliases_share_common_allocator_surface() { + fn specialized_allocator_aliases_share_common_allocator_surface() { let mut je = JeAllocator::with_capacity(32); let ptr = je.Allocate(8).unwrap(); assert!(je.Contains(ptr)); @@ -1334,7 +1367,7 @@ mod tests { } #[test] - fn parity_je_allocator_enforces_capacity_and_tracks_stats() { + fn je_allocator_enforces_capacity_and_tracks_stats() { let mut allocator = JeAllocator::with_capacity(4 * 1024); let ptr = allocator.Allocate(1024).unwrap(); assert!(allocator.Contains(ptr)); @@ -1357,7 +1390,7 @@ mod tests { } #[test] - fn parity_pool_allocator_reuses_fixed_objects_and_tracks_chunks() { + fn pool_allocator_reuses_fixed_objects_and_tracks_chunks() { let mut allocator = PoolBasedMemoryAllocatorDram::new( 1 << 28, PoolBasedMemoryAllocatorBase::DEFAULT_MAX_THREAD_NUM, @@ -1386,10 +1419,10 @@ mod tests { } #[test] - fn parity_pool_allocator_rebalance_exposes_global_free_list_size() { + fn pool_allocator_rebalance_exposes_global_free_list_size() { let mut allocator = PoolBasedMemoryAllocatorPMem::pmem( "/tmp", - FlushPolicy::kNoFlush, + FlushPolicy::NoFlush, 1 << 28, PoolBasedMemoryAllocatorBase::DEFAULT_MAX_THREAD_NUM, PoolBasedMemoryAllocatorBase::DEFAULT_OBJECT_LEN, @@ -1415,7 +1448,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_supports_insert_assign_find_and_erase() { + fn concurrent_hash_map_supports_insert_assign_find_and_erase() { let map = ConcurrentHashMap::::new(2, 4); assert!(map.Empty()); assert_eq!(map.Size(), 0); @@ -1447,7 +1480,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_honors_capacity_and_shared_clones() { + fn concurrent_hash_map_honors_capacity_and_shared_clones() { let map = ConcurrentHashMap::::new(1, 1); assert!(map.Insert(7, "seven".to_string()).unwrap()); assert!(matches!( @@ -1470,7 +1503,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_exposes_at_iterate_and_emplace_surface() { + fn concurrent_hash_map_exposes_at_iterate_and_emplace_surface() { let map = ConcurrentHashMap::::new(2, 16); assert_eq!(map.At(&20), 0); assert_eq!(map.GetOrDefault(&20), 0); @@ -1497,7 +1530,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_erases_by_entry_and_predicate() { + fn concurrent_hash_map_erases_by_entry_and_predicate() { let map = ConcurrentHashMap::::new(3, 0); assert!(map.Insert("live".to_string(), 10).unwrap()); assert!(map.Insert("stale".to_string(), 20).unwrap()); @@ -1520,7 +1553,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_returns_iterator_style_insert_results() { + fn concurrent_hash_map_returns_iterator_style_insert_results() { let map = ConcurrentHashMap::::new(2, 4); let first = map.InsertEntry(1, 10).unwrap(); assert!(first.second); @@ -1552,7 +1585,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_erases_entries_by_snapshot_predicate() { + fn concurrent_hash_map_erases_entries_by_snapshot_predicate() { let map = ConcurrentHashMap::::new(2, 0); for key in 0..10 { assert!(map.Insert(key, key).unwrap()); @@ -1566,7 +1599,7 @@ mod tests { } #[test] - fn parity_hist_stats_reports_percentiles_average_max_and_reset() { + fn hist_stats_reports_percentiles_average_max_and_reset() { let mut stats = HistStats::with_bucket_size(8); for value in [1, 2, 2, 4, 9] { stats.Append(value); @@ -1586,7 +1619,7 @@ mod tests { } #[test] - fn parity_hist_stats_merge_preserves_large_latency_tail() { + fn hist_stats_merge_preserves_large_latency_tail() { let mut left = HistStats::with_bucket_size(4); let mut right = HistStats::with_bucket_size(4); left.Append(1); @@ -1603,7 +1636,7 @@ mod tests { } #[test] - fn parity_test_instance_helpers_target_exact_cache_tiers() { + fn test_instance_helpers_target_exact_cache_tiers() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -1632,7 +1665,7 @@ mod tests { cache .TEST_Insert( - CacheInstanceType::kDRAM, + CacheInstanceType::Dram, memory_key.clone(), b"dram".to_vec(), b"dram".len(), @@ -1640,7 +1673,7 @@ mod tests { .unwrap(); cache .TEST_Insert( - CacheInstanceType::kPMEM, + CacheInstanceType::Pmem, pmem_key.clone(), b"pmem".to_vec(), b"pmem".len(), @@ -1648,7 +1681,7 @@ mod tests { .unwrap(); cache .TEST_Insert( - CacheInstanceType::kSSD, + CacheInstanceType::Ssd, ssd_key.clone(), b"ssd".to_vec(), b"ssd".len(), @@ -1656,7 +1689,7 @@ mod tests { .unwrap(); let memory_handle = cache - .TEST_Acquire(CacheInstanceType::kDRAM, &memory_key) + .TEST_Acquire(CacheInstanceType::Dram, &memory_key) .unwrap() .expect("memory handle"); assert_eq!(memory_handle.tier(), CacheReadTier::Memory); @@ -1664,7 +1697,7 @@ mod tests { cache.Release(memory_handle); let pmem_handle = cache - .TEST_Acquire(CacheInstanceType::kPMEM, &pmem_key) + .TEST_Acquire(CacheInstanceType::Pmem, &pmem_key) .unwrap() .expect("pmem handle"); assert_eq!(pmem_handle.tier(), CacheReadTier::Pmem); @@ -1675,7 +1708,7 @@ mod tests { assert_eq!(cache.stats().pinned_bytes, 0); let ssd_handle = cache - .TEST_Acquire(CacheInstanceType::kSSD, &ssd_key) + .TEST_Acquire(CacheInstanceType::Ssd, &ssd_key) .unwrap() .expect("ssd handle"); assert_eq!(ssd_handle.tier(), CacheReadTier::Ssd); @@ -1686,44 +1719,44 @@ mod tests { assert_eq!(cache.stats().pinned_bytes, 0); assert!(cache - .TEST_Acquire(CacheInstanceType::kPMEM, &memory_key) + .TEST_Acquire(CacheInstanceType::Pmem, &memory_key) .unwrap() .is_none()); assert!(cache - .TEST_Acquire(CacheInstanceType::kDRAM, &pmem_key) + .TEST_Acquire(CacheInstanceType::Dram, &pmem_key) .unwrap() .is_none()); cache - .TEST_Remove(CacheInstanceType::kPMEM, &pmem_key) + .TEST_Remove(CacheInstanceType::Pmem, &pmem_key) .unwrap(); assert!(cache - .TEST_Acquire(CacheInstanceType::kPMEM, &pmem_key) + .TEST_Acquire(CacheInstanceType::Pmem, &pmem_key) .unwrap() .is_none()); assert_eq!(cache.Lookup(&pmem_key).unwrap(), None); cache - .TEST_Remove(CacheInstanceType::kSSD, &ssd_key) + .TEST_Remove(CacheInstanceType::Ssd, &ssd_key) .unwrap(); assert!(cache - .TEST_Acquire(CacheInstanceType::kSSD, &ssd_key) + .TEST_Acquire(CacheInstanceType::Ssd, &ssd_key) .unwrap() .is_none()); assert!(matches!( cache.TEST_Insert( - CacheInstanceType::kUnified, + CacheInstanceType::Unified, CacheKey::string(44, "bad"), b"bad".to_vec(), 3, ), - Err(CacheError::UnsupportedInstance(CacheInstanceType::kUnified)) + Err(CacheError::UnsupportedInstance(CacheInstanceType::Unified)) )); } #[test] - fn parity_test_counter_and_path_helpers_match_unified_cache_surface() { + fn test_counter_and_path_helpers_match_unified_cache_surface() { let ssd_dir = tempfile::tempdir().unwrap(); let pmem_dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache( @@ -1765,7 +1798,7 @@ mod tests { } #[test] - fn parity_style_cache_traits_support_abstract_interface_consumers() { + fn style_cache_traits_support_abstract_interface_consumers() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache(CacheOptions { dram_capacity: 64, @@ -1785,7 +1818,7 @@ mod tests { assert!(cache_api.start_cache()); assert_eq!(cache_api.capacity_cache(), 64); assert_eq!( - cache_api.capacity_for_instance_cache(CacheInstanceType::kDRAM), + cache_api.capacity_for_instance_cache(CacheInstanceType::Dram), 64 ); assert_eq!(cache_api.size_cache(), 0); @@ -1797,10 +1830,10 @@ mod tests { Some(b"trait-value".to_vec()) ); assert!(cache_api.size_cache() > 0); - assert!(cache_api.used_cache(CacheInstanceType::kDRAM) > 0); - cache_api.set_capacity_for_instance_cache(CacheInstanceType::kDRAM, 8); + assert!(cache_api.used_cache(CacheInstanceType::Dram) > 0); + cache_api.set_capacity_for_instance_cache(CacheInstanceType::Dram, 8); assert_eq!( - cache_api.capacity_for_instance_cache(CacheInstanceType::kDRAM), + cache_api.capacity_for_instance_cache(CacheInstanceType::Dram), 8 ); cache_api.set_capacity_cache(4); @@ -1820,7 +1853,7 @@ mod tests { } #[test] - fn parity_style_builder_can_return_boxed_cache_interface() { + fn style_builder_can_return_boxed_cache_interface() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildCacheApi(CacheOptions { dram_capacity: 64, @@ -1837,20 +1870,20 @@ mod tests { assert_eq!(cache.capacity_cache(), 64); assert_eq!( - cache.capacity_for_instance_cache(CacheInstanceType::kDRAM), + cache.capacity_for_instance_cache(CacheInstanceType::Dram), 64 ); cache .insert_cache(key.clone(), b"boxed".to_vec(), b"boxed".len()) .unwrap(); assert_eq!(cache.lookup_cache(&key).unwrap(), Some(b"boxed".to_vec())); - assert!(cache.used_cache(CacheInstanceType::kDRAM) > 0); + assert!(cache.used_cache(CacheInstanceType::Dram) > 0); cache.reset_cache().unwrap(); assert_eq!(cache.lookup_cache(&key).unwrap(), None); } #[test] - fn parity_style_builder_can_return_boxed_zero_copy_interface() { + fn style_builder_can_return_boxed_zero_copy_interface() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCacheApi(CacheOptions { dram_capacity: 64, @@ -1876,7 +1909,7 @@ mod tests { } #[test] - fn parity_style_zero_copy_trait_preserves_pin_lifetime() { + fn style_zero_copy_trait_preserves_pin_lifetime() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache(CacheOptions { dram_capacity: 64, @@ -1907,7 +1940,7 @@ mod tests { } #[test] - fn parity_pascal_case_handle_methods_clone_and_scoped_lookup_pin_safely() { + fn pascal_case_handle_methods_clone_and_scoped_lookup_pin_safely() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache(CacheOptions { dram_capacity: 32, @@ -1982,7 +2015,7 @@ mod tests { } #[test] - fn parity_tiered_insert_uses_value_size_for_dram_admission() { + fn tiered_insert_uses_value_size_for_dram_admission() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -2014,7 +2047,7 @@ mod tests { } #[test] - fn parity_tiered_insert_pinned_uses_value_size_for_dram_handle() { + fn tiered_insert_pinned_uses_value_size_for_dram_handle() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -2055,7 +2088,7 @@ mod tests { } #[test] - fn parity_cache_options_builder_constructs_equivalent_cache() { + fn cache_options_builder_constructs_equivalent_cache() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::build_zero_copy_cache(CacheOptions { dram_capacity: 64, @@ -2109,7 +2142,7 @@ mod tests { } #[test] - fn parity_cache_options_helpers_preserve_documented_policy_names() { + fn cache_options_helpers_preserve_documented_policy_names() { let dir = tempfile::tempdir().unwrap(); let options = CacheOptions::new(32, 96, 256) .with_ssd_paths(vec![dir.path().to_path_buf()]) @@ -2122,31 +2155,31 @@ mod tests { .with_ssd_instance_only(false); assert_eq!( - CacheReplacementPolicy::from_reference_name("FIFO"), + CacheReplacementPolicy::from_config_name("FIFO"), CacheReplacementPolicy::Fifo ); assert_eq!( - CacheReplacementPolicy::from_reference_name("SLRU"), + CacheReplacementPolicy::from_config_name("SLRU"), CacheReplacementPolicy::Slru ); assert_eq!( - CacheDataPlacement::from_reference_name("SideBySide"), + CacheDataPlacement::from_config_name("SideBySide"), CacheDataPlacement::SideBySide ); assert_eq!( - CacheDataPlacement::try_from_reference_name("kSideBySide").unwrap(), + CacheDataPlacement::try_from_config_name("kSideBySide").unwrap(), CacheDataPlacement::SideBySide ); assert_eq!( - CacheDataPlacement::try_from_reference_name("Tiered").unwrap(), + CacheDataPlacement::try_from_config_name("Tiered").unwrap(), CacheDataPlacement::Tiered ); assert_eq!( - DRAMPMEMDataPlacementType::try_from_reference_name("kTiered").unwrap(), - DRAMPMEMDataPlacementType::kTiered + DRAMPMEMDataPlacementType::try_from_config_name("kTiered").unwrap(), + DRAMPMEMDataPlacementType::Tiered ); assert!(matches!( - CacheDataPlacement::try_from_reference_name("bad-placement"), + CacheDataPlacement::try_from_config_name("bad-placement"), Err(CacheError::InvalidConfig(_)) )); assert_eq!(options.cache_dram_replacement_policy, "FIFO"); @@ -2176,7 +2209,7 @@ mod tests { } #[test] - fn parity_multi_tier_cache_rejects_invalid_placement_config() { + fn multi_tier_cache_rejects_invalid_placement_config() { let dir = tempfile::tempdir().unwrap(); let cache = MultiTierCache::try_new( 32, @@ -2228,7 +2261,7 @@ mod tests { } #[test] - fn parity_pascal_case_builder_factories_match_matrixcache_builder_names() { + fn pascal_case_builder_factories_match_matrixcache_builder_names() { let cache_dir = tempfile::tempdir().unwrap(); let zero_copy_dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildCache(CacheOptions { @@ -2271,7 +2304,7 @@ mod tests { } #[test] - fn parity_cache_options_zero_ssd_capacity_disables_ssd_tier() { + fn cache_options_zero_ssd_capacity_disables_ssd_tier() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::build_zero_copy_cache(CacheOptions { dram_capacity: 64, @@ -2596,7 +2629,7 @@ mod tests { let cache = MultiLayerCache::with_options(options.clone()); cache .test_insert( - CacheInstanceType::kPMEM, + CacheInstanceType::Pmem, pmem_key.clone(), b"pmem-value".to_vec(), 10, @@ -2604,7 +2637,7 @@ mod tests { .unwrap(); cache .test_insert( - CacheInstanceType::kSSD, + CacheInstanceType::Ssd, ssd_key.clone(), b"ssd-value".to_vec(), 9, @@ -3081,38 +3114,38 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(64, dir.path()); - assert_eq!(CacheInstanceType::kDRAM as u8, 0); - assert_eq!(CacheInstanceType::kPMEM as u8, 1); - assert_eq!(CacheInstanceType::kSSD as u8, 2); - assert_eq!(CacheInstanceType::kUnified as u8, 3); - assert_eq!(DRAMPMEMDataPlacementType::kSideBySide as u8, 0); - assert_eq!(DRAMPMEMDataPlacementType::kTiered as u8, 1); - assert_eq!(DRAMPMEMDataPlacementType::kMaxCode as u8, 2); + assert_eq!(CacheInstanceType::Dram as u8, 0); + assert_eq!(CacheInstanceType::Pmem as u8, 1); + assert_eq!(CacheInstanceType::Ssd as u8, 2); + assert_eq!(CacheInstanceType::Unified as u8, 3); + assert_eq!(DRAMPMEMDataPlacementType::SideBySide as u8, 0); + assert_eq!(DRAMPMEMDataPlacementType::Tiered as u8, 1); + assert_eq!(DRAMPMEMDataPlacementType::MaxCode as u8, 2); assert_eq!( - DRAMPMEMDataPlacementType::FromReferenceName("kSideBySide"), - DRAMPMEMDataPlacementType::kSideBySide + DRAMPMEMDataPlacementType::FromConfigName("kSideBySide"), + DRAMPMEMDataPlacementType::SideBySide ); assert_eq!( - DRAMPMEMDataPlacementType::kTiered.AsCacheDataPlacement(), + DRAMPMEMDataPlacementType::Tiered.AsCacheDataPlacement(), CacheDataPlacement::Tiered ); assert_eq!(cache.data_placement(), CacheDataPlacement::Tiered); assert_eq!( cache.GetDRAMPMEMDataPlacementType(), - DRAMPMEMDataPlacementType::kTiered + DRAMPMEMDataPlacementType::Tiered ); cache.set_data_placement(CacheDataPlacement::SideBySide); cache.set_data_placement_threshold_bytes(32); assert_eq!(cache.data_placement(), CacheDataPlacement::SideBySide); assert_eq!( - cache.reference_data_placement_type(), - DRAMPMEMDataPlacementType::kSideBySide + cache.config_data_placement_type(), + DRAMPMEMDataPlacementType::SideBySide ); assert_eq!(cache.data_placement_threshold_bytes(), 32); - cache.SetDRAMPMEMDataPlacementType(DRAMPMEMDataPlacementType::kTiered); + cache.SetDRAMPMEMDataPlacementType(DRAMPMEMDataPlacementType::Tiered); assert_eq!(cache.GetDataPlacementType(), CacheDataPlacement::Tiered); } @@ -3257,43 +3290,43 @@ mod tests { } #[test] - fn parity_strict_replacement_policy_setter_is_pre_start_only() { + fn strict_replacement_policy_setter_is_pre_start_only() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(64, dir.path()); assert!(matches!( cache.TrySetReplacementPolicyType( - CacheInstanceType::kDRAM, + CacheInstanceType::Dram, CacheReplacementPolicy::Fifo ), Err(CacheError::AlreadyStarted) )); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::WeightedHotnessLru ); cache.Stop(); cache - .TrySetReplacementPolicyType(CacheInstanceType::kDRAM, CacheReplacementPolicy::Fifo) + .TrySetReplacementPolicyType(CacheInstanceType::Dram, CacheReplacementPolicy::Fifo) .unwrap(); cache .try_set_replacement_policy_for_tier(CacheTier::Pmem, CacheReplacementPolicy::Slru) .unwrap(); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::Fifo ); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kPMEM), + cache.GetReplacementPolicyType(CacheInstanceType::Pmem), CacheReplacementPolicy::Slru ); assert!(matches!( cache.TrySetReplacementPolicyType( - CacheInstanceType::kUnified, + CacheInstanceType::Unified, CacheReplacementPolicy::Fifo ), - Err(CacheError::UnsupportedInstance(CacheInstanceType::kUnified)) + Err(CacheError::UnsupportedInstance(CacheInstanceType::Unified)) )); assert!(cache.Start()); @@ -3351,27 +3384,27 @@ mod tests { } #[test] - fn access_record_type_matches_reference_codes_and_aliases() { - assert_eq!(CacheAccessRecordType::Put.reference_code(), 1); - assert_eq!(CacheAccessRecordType::Get.ReferenceCode(), 2); - assert_eq!(CacheAccessRecordType::Delete.reference_code(), 3); + fn access_record_type_matches_config_codes_and_aliases() { + assert_eq!(CacheAccessRecordType::Put.config_code(), 1); + assert_eq!(CacheAccessRecordType::Get.ConfigCode(), 2); + assert_eq!(CacheAccessRecordType::Delete.config_code(), 3); assert_eq!(CacheAccessRecordType::kPut, CacheAccessRecordType::Put); assert_eq!(AccessRecordType::kGet, CacheAccessRecordType::Get); - assert_eq!(AccessRecordType::kDelete.AsReferenceName(), "kDelete"); + assert_eq!(AccessRecordType::kDelete.AsConfigName(), "kDelete"); assert_eq!(AccessRecordType::kMaxCode, 4); assert_eq!( - AccessRecordType::from_reference_code(1), + AccessRecordType::from_config_code(1), Some(CacheAccessRecordType::Put) ); assert_eq!( - AccessRecordType::FromReferenceCode(2), + AccessRecordType::FromConfigCode(2), Some(CacheAccessRecordType::Get) ); assert_eq!( - AccessRecordType::from_reference_code(3), + AccessRecordType::from_config_code(3), Some(CacheAccessRecordType::Delete) ); - assert_eq!(AccessRecordType::from_reference_code(4), None); + assert_eq!(AccessRecordType::from_config_code(4), None); } #[test] @@ -3397,7 +3430,7 @@ mod tests { } #[test] - fn parity_access_record_callback_aliases_register_and_deregister() { + fn access_record_callback_aliases_register_and_deregister() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(64, dir.path()); let events = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -3496,7 +3529,7 @@ mod tests { } #[test] - fn parity_eviction_handler_aliases_disable_and_reenable_callback_delivery() { + fn eviction_handler_aliases_disable_and_reenable_callback_delivery() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(8, dir.path()); let evictions = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -3532,18 +3565,18 @@ mod tests { } #[test] - fn parity_cache_instance_dram_surface_puts_gets_peeks_deletes_and_resets() { + fn cache_instance_dram_surface_puts_gets_peeks_deletes_and_resets() { let mut instance = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); - assert_eq!(instance.StorageEngineType(), StorageEngineType::kDRAM); - assert_eq!(instance.TEST_GetStorageEngine(), StorageEngineType::kDRAM); + assert_eq!(instance.StorageEngineType(), StorageEngineType::Dram); + assert_eq!(instance.TEST_GetStorageEngine(), StorageEngineType::Dram); assert_eq!( instance.TEST_GetStorageEngineType(), - StorageEngineType::kDRAM + StorageEngineType::Dram ); instance.Start().unwrap(); @@ -3649,11 +3682,11 @@ mod tests { } #[test] - fn parity_cache_instance_put_returning_buffer_matches_put_result_surface() { + fn cache_instance_put_returning_buffer_matches_put_result_surface() { let instance = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); @@ -3671,12 +3704,12 @@ mod tests { } #[test] - fn parity_cache_instance_put_returning_buffer_reports_ssd_tier() { + fn cache_instance_put_returning_buffer_reports_ssd_tier() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![dir.path().to_path_buf()], ); @@ -3693,12 +3726,12 @@ mod tests { } #[test] - fn parity_cache_instance_pmem_surface_uses_exact_pmem_tier() { + fn cache_instance_pmem_surface_uses_exact_pmem_tier() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 32, - ReplacementPolicyType::kSLRU, - StorageEngineType::kPMEM, + ReplacementPolicyType::Slru, + StorageEngineType::Pmem, vec![dir.path().to_path_buf()], ); @@ -3717,18 +3750,18 @@ mod tests { } #[test] - fn parity_l1_cache_implement_pulls_dram_then_pmem_without_replacement_access() { + fn l1_cache_implement_pulls_dram_then_pmem_without_replacement_access() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); let pmem_dir = tempfile::tempdir().unwrap(); let pmem = CacheInstance::new( 64, - ReplacementPolicyType::kSLRU, - StorageEngineType::kPMEM, + ReplacementPolicyType::Slru, + StorageEngineType::Pmem, vec![pmem_dir.path().to_path_buf()], ); dram.Put("shared", b"dram-value".to_vec()).unwrap(); @@ -3756,11 +3789,11 @@ mod tests { } #[test] - fn parity_l1_cache_implement_allows_absent_pmem_instance() { + fn l1_cache_implement_allows_absent_pmem_instance() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); dram.Put("dram-only", b"value".to_vec()).unwrap(); @@ -3780,18 +3813,18 @@ mod tests { } #[test] - fn parity_l2_cache_policy_access_tail_and_write_match_arc_flow() { + fn l2_cache_policy_access_tail_and_write_match_arc_flow() { let dram = CacheInstance::new( 128, - ReplacementPolicyType::kWeightedHotnessLru, - StorageEngineType::kDRAM, + ReplacementPolicyType::WeightedHotnessLru, + StorageEngineType::Dram, vec![], ); let l2_dir = tempfile::tempdir().unwrap(); let l2 = CacheInstance::new( 256, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.path().to_path_buf()], ); dram.Put("cold-a", b"alpha".to_vec()).unwrap(); @@ -3841,18 +3874,18 @@ mod tests { } #[test] - fn parity_l2_cache_policy_eviction_queue_duplicate_and_overflow_paths() { + fn l2_cache_policy_eviction_queue_duplicate_and_overflow_paths() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, vec![], ); let l2_dir = tempfile::tempdir().unwrap(); let l2 = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.path().to_path_buf()], ); let l1 = L1CacheImplement::new(dram, None); @@ -3916,14 +3949,14 @@ mod tests { ) -> L2CachePolicy { let dram = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, vec![], ); let l2 = CacheInstance::new( 256, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.to_path_buf()], ); let l1 = L1CacheImplement::new(dram, None); @@ -3933,7 +3966,7 @@ mod tests { } #[test] - fn parity_l2_cache_policy_access_buffering_modes_and_drop_on_full() { + fn l2_cache_policy_access_buffering_modes_and_drop_on_full() { let dir = tempfile::tempdir().unwrap(); let mut policy = l2_test_policy(dir.path(), 8, 2, 8, 8); policy.Start(); @@ -3962,7 +3995,7 @@ mod tests { } #[test] - fn parity_l2_cache_policy_eviction_handler_is_off_by_default() { + fn l2_cache_policy_eviction_handler_is_off_by_default() { let dir = tempfile::tempdir().unwrap(); let mut policy = l2_test_policy(dir.path(), 8, 8, 8, 8); policy.Start(); @@ -3984,7 +4017,7 @@ mod tests { } #[test] - fn parity_l2_cache_policy_poll_paces_passes_by_interval() { + fn l2_cache_policy_poll_paces_passes_by_interval() { let dir = tempfile::tempdir().unwrap(); let mut policy = l2_test_policy(dir.path(), 8, 8, 8, 8); policy.Start(); @@ -4019,18 +4052,18 @@ mod tests { } #[test] - fn parity_l2_cache_policy_factory_uses_reference_default_sizing() { + fn l2_cache_policy_factory_uses_config_default_sizing() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, vec![], ); let l2_dir = tempfile::tempdir().unwrap(); let l2 = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.path().to_path_buf()], ); let l1 = L1CacheImplement::new(dram, None); @@ -4048,18 +4081,18 @@ mod tests { } #[test] - fn parity_l2_cache_policy_factory_builds_started_policy_surface() { + fn l2_cache_policy_factory_builds_started_policy_surface() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, vec![], ); let l2_dir = tempfile::tempdir().unwrap(); let l2 = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.path().to_path_buf()], ); let l1 = L1CacheImplement::new(dram, None); @@ -4072,17 +4105,17 @@ mod tests { } #[test] - fn parity_cache_instance_ssd_surface_recovers_persistent_index() { + fn cache_instance_ssd_surface_recovers_persistent_index() { let dir = tempfile::tempdir().unwrap(); let paths = vec![dir.path().to_path_buf()]; let instance = CacheInstance::new( 128, - ReplacementPolicyType::kWeightedHotnessLru, - StorageEngineType::kSSD, + ReplacementPolicyType::WeightedHotnessLru, + StorageEngineType::Ssd, paths.clone(), ); - assert_eq!(instance.StorageEngineType(), StorageEngineType::kSSD); - assert_eq!(instance.TEST_GetStorageEngine(), StorageEngineType::kSSD); + assert_eq!(instance.StorageEngineType(), StorageEngineType::Ssd); + assert_eq!(instance.TEST_GetStorageEngine(), StorageEngineType::Ssd); instance.Put("ssd-key", b"ssd-value".to_vec()).unwrap(); assert_eq!( instance.Get("ssd-key").unwrap(), @@ -4097,8 +4130,8 @@ mod tests { let restarted = CacheInstance::new( 128, - ReplacementPolicyType::kWeightedHotnessLru, - StorageEngineType::kSSD, + ReplacementPolicyType::WeightedHotnessLru, + StorageEngineType::Ssd, paths, ); let report = restarted.RecoverData().unwrap(); @@ -4110,12 +4143,12 @@ mod tests { } #[test] - fn parity_cache_instance_ssd_put_bypass_storage_writes_value() { + fn cache_instance_ssd_put_bypass_storage_writes_value() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![dir.path().to_path_buf()], ); @@ -4141,12 +4174,12 @@ mod tests { } #[test] - fn parity_cache_instance_ssd_update_rewrites_guarded_block() { + fn cache_instance_ssd_update_rewrites_guarded_block() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![dir.path().to_path_buf()], ); @@ -4190,11 +4223,11 @@ mod tests { } #[test] - fn parity_cache_instance_eviction_and_metric_handlers_follow_status() { + fn cache_instance_eviction_and_metric_handlers_follow_status() { let instance = CacheInstance::new( 8, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); let evictions = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -4230,7 +4263,7 @@ mod tests { } #[test] - fn parity_cache_buffer_exposes_key_data_size_and_set_key() { + fn cache_buffer_exposes_key_data_size_and_set_key() { let mut buffer = StringBuffer::string("hello"); assert_eq!(buffer.Key(), ""); buffer.SetKey("buffer-key"); @@ -4249,7 +4282,7 @@ mod tests { } #[test] - fn parity_iobuf_buffer_owns_data_and_converts_to_cache_buffer() { + fn iobuf_buffer_owns_data_and_converts_to_cache_buffer() { let mut buffer = IOBufBuffer::new(b"iobuf-value".to_vec()); assert_eq!(buffer.Key(), ""); buffer.SetKey("iobuf-key"); @@ -4265,8 +4298,8 @@ mod tests { let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); let inserted = instance.PutBuffer(converted).unwrap(); @@ -4278,9 +4311,9 @@ mod tests { } #[test] - fn parity_raw_buffer_exposes_owned_data_reset_and_cache_buffer_conversion() { + fn raw_buffer_exposes_owned_data_reset_and_cache_buffer_conversion() { let mut raw = - RawBuffer::with_storage_engine(b"raw-value".to_vec(), StorageEngineType::kPMEM, true); + RawBuffer::with_storage_engine(b"raw-value".to_vec(), StorageEngineType::Pmem, true); assert_eq!(raw.Key(), ""); raw.SetKey("raw-key"); assert_eq!(raw.Key(), "raw-key"); @@ -4288,7 +4321,7 @@ mod tests { assert_eq!(raw.DataPtr(), raw.Data().as_ptr()); assert_eq!(raw.Value(), b"raw-value"); assert_eq!(raw.Size(), 9); - assert_eq!(raw.storage_engine(), Some(StorageEngineType::kPMEM)); + assert_eq!(raw.storage_engine(), Some(StorageEngineType::Pmem)); assert!(raw.async_delete()); let converted: CacheBuffer = raw.into(); @@ -4308,7 +4341,7 @@ mod tests { } #[test] - fn parity_string_view_buffer_tracks_key_and_size_without_holding_data() { + fn string_view_buffer_tracks_key_and_size_without_holding_data() { let mut view = StringViewBuffer::new(4096); assert_eq!(view.Key(), ""); view.SetKey("ssd-view"); @@ -4324,11 +4357,11 @@ mod tests { } #[test] - fn parity_cache_instance_accepts_raw_buffer_conversion_for_put_buffer() { + fn cache_instance_accepts_raw_buffer_conversion_for_put_buffer() { let instance = CacheInstance::new( 32, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); let mut raw = RawBuffer::new(b"raw-put".to_vec()); @@ -4344,11 +4377,11 @@ mod tests { } #[test] - fn parity_cache_instance_buffer_put_get_and_update_are_guarded() { + fn cache_instance_buffer_put_get_and_update_are_guarded() { let mut instance = CacheInstance::new( 32, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); @@ -4427,7 +4460,7 @@ mod tests { } #[test] - fn parity_flexible_cache_wraps_configurable_cache_instance_for_strings() { + fn flexible_cache_wraps_configurable_cache_instance_for_strings() { let cache = MatrixCacheBuilder::BuildFlexibleCache( 8, "fifo", @@ -4436,8 +4469,8 @@ mod tests { Vec::::new(), ); - assert_eq!(cache.policy(), ReplacementPolicyType::kFIFO); - assert_eq!(cache.engine(), StorageEngineType::kDRAM); + assert_eq!(cache.policy(), ReplacementPolicyType::Fifo); + assert_eq!(cache.engine(), StorageEngineType::Dram); assert!(cache.Start()); cache.Insert("first", "12345678".to_string(), 8).unwrap(); assert_eq!(cache.Lookup("first").unwrap(), Some("12345678".to_string())); @@ -4456,7 +4489,7 @@ mod tests { } #[test] - fn parity_blockcache_facade_enforces_lifecycle_and_clears_ssd_paths() { + fn blockcache_facade_enforces_lifecycle_and_clears_ssd_paths() { let dir = tempfile::tempdir().unwrap(); let ssd_path = dir.path().join("blockcache-ssd"); fs::create_dir_all(&ssd_path).unwrap(); @@ -4492,7 +4525,7 @@ mod tests { } #[test] - fn parity_flexible_cache_uses_selected_ssd_paths_and_recovers() { + fn flexible_cache_uses_selected_ssd_paths_and_recovers() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().to_string_lossy().to_string(); let cache = MatrixCacheBuilder::BuildFlexibleCacheFromPathStrings( @@ -4503,8 +4536,8 @@ mod tests { vec![path.clone()], ); - assert_eq!(cache.policy(), ReplacementPolicyType::kWeightedHotnessLru); - assert_eq!(cache.engine(), StorageEngineType::kSSD); + assert_eq!(cache.policy(), ReplacementPolicyType::WeightedHotnessLru); + assert_eq!(cache.engine(), StorageEngineType::Ssd); assert_eq!(cache.paths(), &[PathBuf::from(&path)]); cache.Insert("ssd-key", "ssd-value".to_string(), 9).unwrap(); assert_eq!( @@ -4529,7 +4562,7 @@ mod tests { } #[test] - fn parity_pmem_cache_instance_persists_and_recovers_from_configured_path() { + fn pmem_cache_instance_persists_and_recovers_from_configured_path() { let dir = tempfile::tempdir().unwrap(); let pmem_path = dir.path().join("pmem-device"); let paths = vec![pmem_path.to_string_lossy().to_string()]; @@ -4540,7 +4573,7 @@ mod tests { paths.clone(), Vec::::new(), ); - assert_eq!(cache.engine(), StorageEngineType::kPMEM); + assert_eq!(cache.engine(), StorageEngineType::Pmem); cache.Insert("pmem-a", "value-a".to_string(), 7).unwrap(); cache.Insert("pmem-b", "value-b".to_string(), 7).unwrap(); assert_eq!(cache.Lookup("pmem-a").unwrap(), Some("value-a".to_string())); @@ -4591,14 +4624,14 @@ mod tests { .with_auto_recover_on_start(true); let cache = MultiLayerCache::try_with_options(options.clone()).unwrap(); cache - .test_insert(CacheInstanceType::kPMEM, key.clone(), b"pmem".to_vec(), 4) + .test_insert(CacheInstanceType::Pmem, key.clone(), b"pmem".to_vec(), 4) .unwrap(); let restarted = MultiLayerCache::try_with_options(options).unwrap(); assert_eq!(restarted.peek_tier(&key), Some(CacheReadTier::Pmem)); assert_eq!( restarted - .test_acquire(CacheInstanceType::kPMEM, &key) + .test_acquire(CacheInstanceType::Pmem, &key) .unwrap() .unwrap() .value(), @@ -4616,11 +4649,11 @@ mod tests { let key = CacheKey::string(7, "general-remove-pmem"); cache - .test_insert(CacheInstanceType::kPMEM, key.clone(), b"pmem".to_vec(), 4) + .test_insert(CacheInstanceType::Pmem, key.clone(), b"pmem".to_vec(), 4) .unwrap(); assert_eq!( cache - .test_acquire(CacheInstanceType::kPMEM, &key) + .test_acquire(CacheInstanceType::Pmem, &key) .unwrap() .unwrap() .value(), @@ -4633,13 +4666,13 @@ mod tests { let report = restarted.recover_pmem_index().unwrap(); assert_eq!(report.recovered_files, 0); assert!(restarted - .test_acquire(CacheInstanceType::kPMEM, &key) + .test_acquire(CacheInstanceType::Pmem, &key) .unwrap() .is_none()); } #[test] - fn parity_flexible_cache_multi_ssd_uses_all_paths_and_recovers() { + fn flexible_cache_multi_ssd_uses_all_paths_and_recovers() { let dir_a = tempfile::tempdir().unwrap(); let dir_b = tempfile::tempdir().unwrap(); let path_a = dir_a.path().to_string_lossy().to_string(); @@ -4653,7 +4686,7 @@ mod tests { paths.clone(), ); - assert_eq!(cache.engine(), StorageEngineType::kMultiSSD); + assert_eq!(cache.engine(), StorageEngineType::MultiSsd); assert_eq!( cache.paths(), &[PathBuf::from(&path_a), PathBuf::from(&path_b)] @@ -4720,7 +4753,7 @@ mod tests { } #[test] - fn parity_multi_tier_cache_wrapper_builds_unified_cache_from_constructor_knobs() { + fn multi_tier_cache_wrapper_builds_unified_cache_from_constructor_knobs() { let pmem_dir = tempfile::tempdir().unwrap(); let ssd_dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildMultiTierCacheFromPathStrings( @@ -4736,8 +4769,8 @@ mod tests { "rocksdb", ); - assert_eq!(cache.policy(), ReplacementPolicyType::kFIFO); - assert_eq!(cache.ssd_storage_engine(), StorageEngineType::kSSD); + assert_eq!(cache.policy(), ReplacementPolicyType::Fifo); + assert_eq!(cache.ssd_storage_engine(), StorageEngineType::Ssd); assert!(!cache.eviction_enabled()); assert_eq!(cache.options().dram_capacity, 16); assert_eq!(cache.options().pmem_capacity, 32); @@ -4747,9 +4780,9 @@ mod tests { "SideBySide" ); assert_eq!(cache.options().cache_dram_pmem_data_placement_threshold, 8); - assert_eq!(cache.inner().GetCapacity(CacheInstanceType::kDRAM), 16); - assert_eq!(cache.inner().GetCapacity(CacheInstanceType::kPMEM), 32); - assert_eq!(cache.inner().GetCapacity(CacheInstanceType::kSSD), 128); + assert_eq!(cache.inner().GetCapacity(CacheInstanceType::Dram), 16); + assert_eq!(cache.inner().GetCapacity(CacheInstanceType::Pmem), 32); + assert_eq!(cache.inner().GetCapacity(CacheInstanceType::Ssd), 128); assert!(!cache.inner().EvictionHandlerEnabled()); assert!(cache.Start()); @@ -5459,6 +5492,111 @@ mod tests { assert_eq!(remaining.iter().filter(|value| value.is_some()).count(), 2); } + /// Every tier's key order has to hold exactly the keys that tier holds. + /// + /// Victim selection reads the order rather than the tier map, so a key the + /// order has lost can never be evicted and a key it lists but the tier no + /// longer holds wastes a round of selection. Neither shows up as a wrong + /// answer until the cache is under pressure, so this checks the invariant + /// directly after a workload that exercises every path that edits it. + #[test] + fn tier_orders_hold_exactly_the_keys_their_tiers_hold() { + let dir = tempfile::tempdir().unwrap(); + let cache = MultiLayerCache::with_options(CacheOptions { + dram_capacity: 512, + pmem_capacity: 512, + ssd_capacity: 4096, + ssd_paths: vec![dir.path().to_path_buf()], + ..CacheOptions::default() + }); + cache.start().unwrap(); + + let keys = (0..64) + .map(|index| CacheKey::string(0, &format!("order-invariant-{index:04}"))) + .collect::>(); + + // Fill past capacity so entries evict and demote between tiers. + for key in &keys { + cache.put(key.clone(), vec![b'z'; 16]).unwrap(); + } + // Reads promote and refill, which rewrites tier membership. + for key in keys.iter().step_by(3) { + let _ = cache.get(key).unwrap(); + } + // Explicit removals and an invalidation take their own paths. + for key in keys.iter().step_by(7) { + cache.remove(key).unwrap(); + } + cache.invalidate(&keys[1]).unwrap(); + cache.invalidate_memory_only(&keys[2]); + // A shrink evicts a batch in one pass. + cache.set_capacity_for_tier(CacheTier::Memory, 128); + + let inner = cache.inner.read().expect("cache lock poisoned"); + let order_keys = |order: &CacheKeyOrder| { + order.iter().cloned().collect::>() + }; + assert_eq!( + order_keys(&inner.memory_order), + inner.memory.keys().cloned().collect::>(), + "memory order and memory tier disagree" + ); + assert_eq!( + order_keys(&inner.pmem_order), + inner.pmem.keys().cloned().collect::>(), + "pmem order and pmem tier disagree" + ); + assert_eq!( + order_keys(&inner.disk_order), + inner.disk_index.keys().cloned().collect::>(), + "disk order and disk tier disagree" + ); + } + + /// First-in first-out eviction has to keep going until the tier is back + /// under its budget, however many entries that takes. + /// + /// Selection reads the tier's key order, so a victim that is taken but + /// left in that order gets picked again on the next round. The second pick + /// frees nothing, the loop reads that as no progress and stops, and the + /// tier is left over capacity with one entry removed instead of three. + #[test] + fn fifo_capacity_shrink_evicts_the_whole_overage_in_one_pass() { + let dir = tempfile::tempdir().unwrap(); + // Memory only: a victim with a tier below it is demoted rather than + // dropped, and would still read back. + let cache = MultiLayerCache::with_options(CacheOptions { + dram_capacity: 40, + pmem_capacity: 0, + ssd_capacity: 0, + ssd_paths: vec![dir.path().to_path_buf()], + cache_dram_replacement_policy: "FIFO".to_string(), + ..CacheOptions::default() + }); + cache.start().unwrap(); + + let keys = (0..5) + .map(|index| CacheKey::string(0, &format!("fifo-shrink-{index}"))) + .collect::>(); + for key in &keys { + cache.put(key.clone(), vec![b'x'; 8]).unwrap(); + } + assert_eq!(cache.size_for_tier(CacheTier::Memory), 40); + + // Room for two of the five entries, so three have to go at once. + cache.set_capacity_for_tier(CacheTier::Memory, 16); + + assert!( + cache.size_for_tier(CacheTier::Memory) <= 16, + "tier left over capacity at {} bytes", + cache.size_for_tier(CacheTier::Memory) + ); + let remaining = cache.get_batch(&keys).unwrap(); + assert_eq!(remaining.iter().filter(|value| value.is_some()).count(), 2); + // First in, first out: the two survivors are the ones written last. + assert!(remaining[3].is_some() && remaining[4].is_some()); + } + // shared-corpus: storage_cache_refill; #[test] fn weighted_ssd_eviction_preserves_hot_entries() { @@ -5827,7 +5965,7 @@ mod tests { } #[test] - fn pinned_handle_clone_with_cache_matches_reference_explicit_clone_semantics() { + fn pinned_handle_clone_with_cache_matches_config_explicit_clone_semantics() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(8, dir.path()); let key = CacheKey::string(1, "clone"); @@ -5918,7 +6056,7 @@ mod tests { } #[test] - fn scoped_lookup_matches_reference_found_and_auto_release_semantics() { + fn scoped_lookup_matches_config_found_and_auto_release_semantics() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(16, dir.path()); let key = CacheKey::string(1, "scoped-lookup"); @@ -6015,7 +6153,7 @@ mod tests { } #[test] - fn parity_remove_keeps_removed_pinned_entry_counted_until_release() { + fn remove_keeps_removed_pinned_entry_counted_until_release() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(16, dir.path()); let key = CacheKey::page_with_slot(1, 10, 0, 4, Some(7)); @@ -6362,7 +6500,7 @@ mod tests { } #[test] - fn parity_base_lru_list_tracks_mru_and_tail_eviction() { + fn base_lru_list_tracks_mru_and_tail_eviction() { let mut list = BaseLRUList::new(2); list.Put("a".to_string()); list.Put("b".to_string()); @@ -6379,7 +6517,7 @@ mod tests { } #[test] - fn parity_ghost_lru_list_downgrades_data_to_ghost_tail() { + fn ghost_lru_list_downgrades_data_to_ghost_tail() { let mut list = GhostLRUList::new(1); list.Put("hot".to_string()); list.Put("cold".to_string()); @@ -6396,7 +6534,7 @@ mod tests { } #[test] - fn parity_arc_list_promotes_hits_and_keeps_bounded_data_size() { + fn arc_list_promotes_hits_and_keeps_bounded_data_size() { let mut arc = ArcList::new(2); arc.Put("a".to_string()); arc.Put("b".to_string()); @@ -6418,7 +6556,7 @@ mod tests { } #[test] - fn parity_arc_list_hit_on_fetch_data_promotes_to_active() { + fn arc_list_hit_on_fetch_data_promotes_to_active() { let mut arc = ArcList::new(4); arc.Put("a".to_string()); assert_eq!(arc.GetFetchDataTail(8), vec!["a".to_string()]); @@ -6432,7 +6570,7 @@ mod tests { } #[test] - fn parity_arc_list_ghost_hits_adapt_the_fetch_active_split() { + fn arc_list_ghost_hits_adapt_the_fetch_active_split() { // Capacity 2 starts split evenly, one slot each side. let mut arc = ArcList::new(2); assert_eq!(arc.FetchCapacity(), 1); @@ -6463,7 +6601,7 @@ mod tests { } #[test] - fn parity_arc_list_drops_fetch_tail_outright_when_its_ghost_is_empty() { + fn arc_list_drops_fetch_tail_outright_when_its_ghost_is_empty() { let mut arc = ArcList::new(2); arc.Put("a".to_string()); arc.Put("b".to_string()); @@ -6503,7 +6641,7 @@ mod tests { } #[test] - fn parity_replacement_arc_exposes_active_and_fetch_tail_surface() { + fn replacement_arc_exposes_active_and_fetch_tail_surface() { let mut policy = ReplacementArc::new(2); assert!(!policy.is_initialized()); policy.Init().unwrap(); @@ -6528,55 +6666,55 @@ mod tests { } #[test] - fn parity_storage_engine_type_preserves_codes_and_aliases() { + fn storage_engine_type_preserves_codes_and_aliases() { assert_eq!( - StorageEngineType::from_reference_name("kDRAMStorageEngine"), - StorageEngineType::kDRAM + StorageEngineType::from_config_name("kDRAMStorageEngine"), + StorageEngineType::Dram ); assert_eq!( - StorageEngineType::from_reference_name("kSimpleStorageEngine"), - StorageEngineType::kSimple + StorageEngineType::from_config_name("kSimpleStorageEngine"), + StorageEngineType::Simple ); - assert_eq!(StorageEngineType::kMultiSSD.ReferenceCode(), 4); + assert_eq!(StorageEngineType::MultiSsd.ConfigCode(), 4); assert_eq!( - StorageEngineType::kSimple.AsReferenceEnumName(), + StorageEngineType::Simple.AsConfigEnumName(), "kSimpleStorageEngine" ); assert_eq!( - StorageEngineType::from_reference_name("rocksdb"), - StorageEngineType::kSSD + StorageEngineType::from_config_name("rocksdb"), + StorageEngineType::Ssd ); assert_eq!( - StorageEngineType::from_reference_name("kSSDRocksDBStorageEngine"), - StorageEngineType::kSSD + StorageEngineType::from_config_name("kSSDRocksDBStorageEngine"), + StorageEngineType::Ssd ); - assert_eq!(SSDEngineType::kRocksDB as u8, 0); + assert_eq!(SSDEngineType::RocksDb as u8, 0); assert_eq!( - SSDEngineType::FromReferenceName("rocksdb"), - SSDEngineType::kRocksDB + SSDEngineType::FromConfigName("rocksdb"), + SSDEngineType::RocksDb ); - assert_eq!(SSDEngineType::kRocksDB.AsReferenceName(), "RocksDB"); - assert_eq!(WriteBufferType::kUserDataBuf as u8, 0); - assert_eq!(WriteBufferType::kMetaDataBuf as u8, 1); - assert_eq!(WriteBufferType::kGCBuf as u8, 2); - assert_eq!(WriteBufferType::kCodecDataBuf as u8, 3); - assert_eq!(DataType::DATA as u8, 1); - assert_eq!(DataType::META_LOG as u8, 2); - assert_eq!(GCMode::LOSSY as u8, 1); - assert_eq!(GCMode::LOSSLESS as u8, 10); - assert_eq!(RecordStateType::kSoftDel as u8, 0x0); - assert_eq!(RecordStateType::kNormal as u8, 0x1); - assert_eq!(RecordStateType::kPinned as u8, 0x2); - assert_eq!(RecordStateType::kMaxCode as u8, 0xf); + assert_eq!(SSDEngineType::RocksDb.AsConfigName(), "RocksDB"); + assert_eq!(WriteBufferType::UserDataBuf as u8, 0); + assert_eq!(WriteBufferType::MetaDataBuf as u8, 1); + assert_eq!(WriteBufferType::GcBuf as u8, 2); + assert_eq!(WriteBufferType::CodecDataBuf as u8, 3); + assert_eq!(DataType::Data as u8, 1); + assert_eq!(DataType::MetaLog as u8, 2); + assert_eq!(GCMode::Lossy as u8, 1); + assert_eq!(GCMode::Lossless as u8, 10); + assert_eq!(RecordStateType::SoftDel as u8, 0x0); + assert_eq!(RecordStateType::Normal as u8, 0x1); + assert_eq!(RecordStateType::Pinned as u8, 0x2); + assert_eq!(RecordStateType::MaxCode as u8, 0xf); } #[test] - fn parity_write_buffer_and_encoder_preserve_layout_size_semantics() { - let mut buffer = WriteBuffer::new(WriteBufferType::kUserDataBuf, 128); + fn write_buffer_and_encoder_preserve_layout_size_semantics() { + let mut buffer = WriteBuffer::new(WriteBufferType::UserDataBuf, 128); buffer.PushBack("a", b"one".to_vec()); buffer.PushBack("bb", b"twotwo".to_vec()); assert_eq!(buffer.Capacity(), 128); - assert_eq!(buffer.BufType(), WriteBufferType::kUserDataBuf); + assert_eq!(buffer.BufType(), WriteBufferType::UserDataBuf); assert_eq!(buffer.Count(), 2); assert_eq!(buffer.KeySize(), 3); assert_eq!(buffer.ValueSize(), 9); @@ -6609,7 +6747,7 @@ mod tests { } #[test] - fn parity_mem_storage_crc_is_castagnoli_and_covers_the_length_header() { + fn mem_storage_crc_is_castagnoli_and_covers_the_length_header() { // Pin the algorithm to its published check value rather than to // whatever this implementation happens to emit. assert_eq!(crc32c(b"123456789"), 0xe306_9283); @@ -6643,13 +6781,13 @@ mod tests { } #[test] - fn parity_mem_storage_layout_round_trips_key_value_and_crc() { + fn mem_storage_layout_round_trips_key_value_and_crc() { let crc = MemStorage::ComputeCRC("layout-key", b"layout-value"); let record = MemStorage::DoPutWithCRC("layout-key", b"layout-value", crc).unwrap(); assert_eq!(MemStorage::GetKeyFromData(&record).unwrap(), "layout-key"); let buffer = - MemStorage::CreateCacheBufferFromData(&record, StorageEngineType::kSimple, false) + MemStorage::CreateCacheBufferFromData(&record, StorageEngineType::Simple, false) .unwrap(); assert_eq!(buffer.Key(), "layout-key"); assert_eq!(buffer.Data(), b"layout-value"); @@ -6657,7 +6795,7 @@ mod tests { } #[test] - fn parity_mem_storage_allocator_handle_models_payload_pointer_and_delete() { + fn mem_storage_allocator_handle_models_payload_pointer_and_delete() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(256); let handle = MemStorage::DoPutToAllocator(&mut allocator, "alloc-key", b"alloc-value").unwrap(); @@ -6685,7 +6823,7 @@ mod tests { let buffer = MemStorage::CreateCacheBufferFromAllocatorData( &allocator, handle, - StorageEngineType::kSimple, + StorageEngineType::Simple, false, ) .unwrap(); @@ -6701,7 +6839,7 @@ mod tests { } #[test] - fn parity_mem_storage_allocator_path_rejects_corrupt_crc_before_write() { + fn mem_storage_allocator_path_rejects_corrupt_crc_before_write() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(256); let crc = MemStorage::ComputeCRC("alloc-key", b"alloc-value"); @@ -6716,7 +6854,7 @@ mod tests { } #[test] - fn parity_mem_storage_allocator_surface_works_for_je_and_pool_allocators() { + fn mem_storage_allocator_surface_works_for_je_and_pool_allocators() { let mut je = JeAllocator::with_capacity(256); let je_handle = MemStorage::DoPutToAllocator(&mut je, "je-key", b"je-value").unwrap(); assert_eq!(je_handle.PayloadOffset(), MemStorage::HEADER_BYTES); @@ -6738,7 +6876,7 @@ mod tests { } #[test] - fn parity_simple_storage_engine_lifecycle_put_peek_delete_and_recover() { + fn simple_storage_engine_lifecycle_put_peek_delete_and_recover() { #[derive(Default)] struct Collector { recovered: Vec<(String, Vec)>, @@ -6752,7 +6890,7 @@ mod tests { let mut engine = StorageEngineSimple::with_capacity(1024); assert_eq!(engine.Capacity(), 1024); - assert_eq!(engine.StorageEngineType(), StorageEngineType::kSimple); + assert_eq!(engine.StorageEngineType(), StorageEngineType::Simple); engine.SetCapacity(2048); assert_eq!(engine.Capacity(), 2048); assert!(!engine.is_started()); @@ -6817,7 +6955,7 @@ mod tests { let mut engine = StorageEngineRocksDB::new(&db_path_str); assert_eq!(engine.Path(), db_path_str); - assert_eq!(engine.StorageEngineType(), StorageEngineType::kSSD); + assert_eq!(engine.StorageEngineType(), StorageEngineType::Ssd); assert_eq!( engine.SsdBackendName(), if cfg!(feature = "rocksdb-ssd") { @@ -6912,7 +7050,7 @@ mod tests { } #[test] - fn parity_storage_recover_callback_mock_tracks_last_key_and_count() { + fn storage_recover_callback_mock_tracks_last_key_and_count() { let mut engine = StorageEngineSimple::with_capacity(1024); assert!(engine.Start()); engine.Put("first", b"111".to_vec()).unwrap(); @@ -6945,7 +7083,7 @@ mod tests { } #[test] - fn parity_gc_copy_callback_mock_replaces_buffers_with_guarded_old_data() { + fn gc_copy_callback_mock_replaces_buffers_with_guarded_old_data() { let mut callback = GCCopyCallbackMock::new(); let mut old = CacheBuffer::new(b"old-value".to_vec()); old.SetKey("alpha"); @@ -6984,7 +7122,7 @@ mod tests { } #[test] - fn parity_log_allocator_gc_listener_mock_updates_maps_and_frees_old_ptr() { + fn log_allocator_gc_listener_mock_updates_maps_and_frees_old_ptr() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); let old_ptr = allocator.Allocate(8).unwrap(); let new_ptr = allocator.Allocate(8).unwrap(); @@ -7016,7 +7154,7 @@ mod tests { } #[test] - fn parity_pmem_recover_listener_dedupes_records_before_callback() { + fn pmem_recover_listener_dedupes_records_before_callback() { #[derive(Default)] struct Collector { recovered: Vec<(String, Vec)>, @@ -7049,7 +7187,7 @@ mod tests { } #[test] - fn parity_pmem_storage_test_hooks_put_to_numa_and_report_recover_stats() { + fn pmem_storage_test_hooks_put_to_numa_and_report_recover_stats() { let mut engine = StorageEnginePMem::with_capacity(1024); assert!(engine.Start()); engine.TEST_JoinPmemWriteExecutor(); @@ -7068,12 +7206,12 @@ mod tests { } #[test] - fn parity_ssd_fifo_keeps_insertion_order_when_a_key_is_rewritten() { + fn ssd_fifo_keeps_insertion_order_when_a_key_is_rewritten() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 520, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![dir.path().to_path_buf()], ); instance.Start().unwrap(); @@ -7102,7 +7240,7 @@ mod tests { } #[test] - fn parity_multi_ssd_selects_the_device_with_the_shared_hash() { + fn multi_ssd_selects_the_device_with_the_shared_hash() { let dir = tempfile::tempdir().unwrap(); let dev = |name: &str| dir.path().join(name).to_string_lossy().to_string(); let paths = vec![dev("ssd-a"), dev("ssd-b"), dev("ssd-c")]; @@ -7134,7 +7272,7 @@ mod tests { } #[test] - fn parity_multi_ssd_requires_devices_and_hashes_keys_to_storage() { + fn multi_ssd_requires_devices_and_hashes_keys_to_storage() { let mut empty = StorageEngineMultiSSD::new(Vec::::new(), 1024); assert!(!empty.Start()); assert!(matches!( @@ -7170,7 +7308,7 @@ mod tests { } #[test] - fn parity_multi_ssd_recovers_resets_and_manages_devices() { + fn multi_ssd_recovers_resets_and_manages_devices() { struct Collector { recovered: Vec<(String, Vec)>, } @@ -7184,7 +7322,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let dev = |name: &str| dir.path().join(name).to_string_lossy().to_string(); let mut engine = StorageEngineMultiSSD::new(vec![dev("ssd-a"), dev("ssd-b")], 2048); - assert_eq!(engine.StorageEngineType(), StorageEngineType::kMultiSSD); + assert_eq!(engine.StorageEngineType(), StorageEngineType::MultiSsd); assert!(engine.Start()); engine.Put("first", b"111".to_vec()).unwrap(); engine.Put("second", b"222".to_vec()).unwrap(); @@ -7327,26 +7465,26 @@ mod tests { #[test] fn storage_config_storage_engine_type_conversions() { use StorageEngineType::*; - for ty in [kDRAM, kPMEM, kSSD, kSimple, kMultiSSD] { + for ty in [Dram, Pmem, Ssd, Simple, MultiSsd] { // code conversion round-trips, and every variant has a display name - assert_eq!(StorageEngineType::from_reference_code(ty.reference_code()), ty); - assert!(!ty.as_reference_name().is_empty()); + assert_eq!(StorageEngineType::from_config_code(ty.config_code()), ty); + assert!(!ty.as_config_name().is_empty()); } // recognized name spellings parse to the expected engine - assert_eq!(StorageEngineType::from_reference_name("ssd"), kSSD); - assert_eq!(StorageEngineType::from_reference_name("kRocksDB"), kSSD); - assert_eq!(StorageEngineType::from_reference_name("pmem"), kPMEM); - assert_eq!(StorageEngineType::from_reference_name("multi_ssd"), kMultiSSD); - assert_eq!(StorageEngineType::from_reference_name("simple"), kSimple); - assert_eq!(StorageEngineType::from_reference_name("dram"), kDRAM); - assert!(kSSD.is_ssd_like()); - assert!(kMultiSSD.is_ssd_like()); - assert!(!kDRAM.is_ssd_like()); - assert_eq!(kPMEM.canonical_instance_type(), CacheInstanceType::kPMEM); - assert_eq!(kDRAM.canonical_instance_type(), CacheInstanceType::kDRAM); + assert_eq!(StorageEngineType::from_config_name("ssd"), Ssd); + assert_eq!(StorageEngineType::from_config_name("kRocksDB"), Ssd); + assert_eq!(StorageEngineType::from_config_name("pmem"), Pmem); + assert_eq!(StorageEngineType::from_config_name("multi_ssd"), MultiSsd); + assert_eq!(StorageEngineType::from_config_name("simple"), Simple); + assert_eq!(StorageEngineType::from_config_name("dram"), Dram); + assert!(Ssd.is_ssd_like()); + assert!(MultiSsd.is_ssd_like()); + assert!(!Dram.is_ssd_like()); + assert_eq!(Pmem.canonical_instance_type(), CacheInstanceType::Pmem); + assert_eq!(Dram.canonical_instance_type(), CacheInstanceType::Dram); // unknown code and name fall back to the default engine - assert_eq!(StorageEngineType::from_reference_code(200), kDRAM); - assert_eq!(StorageEngineType::from_reference_name("not-a-real-engine"), kDRAM); + assert_eq!(StorageEngineType::from_config_code(200), Dram); + assert_eq!(StorageEngineType::from_config_name("not-a-real-engine"), Dram); } #[test] @@ -7373,10 +7511,10 @@ mod tests { index.ScanIndexForRecover(|_key, _value| scanned += 1); assert!(scanned >= 1); - let mut wb = WriteBuffer::new(WriteBufferType::kUserDataBuf, 1024); + let mut wb = WriteBuffer::new(WriteBufferType::UserDataBuf, 1024); assert_eq!(wb.Capacity(), 1024); assert_eq!(wb.Count(), 0); - assert_eq!(wb.BufType(), WriteBufferType::kUserDataBuf); + assert_eq!(wb.BufType(), WriteBufferType::UserDataBuf); wb.PushBack("k1", b"v1".to_vec()); wb.PushBack("k2", b"v22".to_vec()); assert_eq!(wb.Count(), 2); @@ -7390,16 +7528,16 @@ mod tests { #[test] fn rdma_utils_and_policy_conversions() { assert_eq!( - RdmaReplacementPolicyType::FIFO.as_replacement_policy_type(), - ReplacementPolicyType::kFIFO + RdmaReplacementPolicyType::Fifo.as_replacement_policy_type(), + ReplacementPolicyType::Fifo ); assert_eq!( - RdmaReplacementPolicyType::LRU.as_replacement_policy_type(), - ReplacementPolicyType::kLRU + RdmaReplacementPolicyType::Lru.as_replacement_policy_type(), + ReplacementPolicyType::Lru ); assert_eq!( - RdmaReplacementPolicyType::OTHER.as_replacement_policy_type(), - ReplacementPolicyType::kMaxCode + RdmaReplacementPolicyType::Other.as_replacement_policy_type(), + ReplacementPolicyType::MaxCode ); let mut generator = RandomStringGenerator::new(); @@ -7419,44 +7557,44 @@ mod tests { CacheAccessRecordType::Get, CacheAccessRecordType::Delete, ] { - assert_eq!(CacheAccessRecordType::from_reference_code(ty.reference_code()), Some(ty)); + assert_eq!(CacheAccessRecordType::from_config_code(ty.config_code()), Some(ty)); } - assert_eq!(CacheAccessRecordType::from_reference_code(0), None); + assert_eq!(CacheAccessRecordType::from_config_code(0), None); assert_eq!( - CacheAccessRecordType::from_reference_code(CacheAccessRecordType::kMaxCode), + CacheAccessRecordType::from_config_code(CacheAccessRecordType::kMaxCode), None ); // CacheDataPlacement name parsing (fallible) assert_eq!( - CacheDataPlacement::try_from_reference_name("SideBySide").unwrap(), + CacheDataPlacement::try_from_config_name("SideBySide").unwrap(), CacheDataPlacement::SideBySide ); assert_eq!( - CacheDataPlacement::try_from_reference_name("Tiered").unwrap(), + CacheDataPlacement::try_from_config_name("Tiered").unwrap(), CacheDataPlacement::Tiered ); - assert!(CacheDataPlacement::try_from_reference_name("nonsense").is_err()); + assert!(CacheDataPlacement::try_from_config_name("nonsense").is_err()); // DRAMPMEMDataPlacementType conversions to/from CacheDataPlacement and names assert_eq!( DRAMPMEMDataPlacementType::from_cache_data_placement(CacheDataPlacement::SideBySide), - DRAMPMEMDataPlacementType::kSideBySide + DRAMPMEMDataPlacementType::SideBySide ); assert_eq!( DRAMPMEMDataPlacementType::from_cache_data_placement(CacheDataPlacement::Tiered), - DRAMPMEMDataPlacementType::kTiered + DRAMPMEMDataPlacementType::Tiered ); assert_eq!( - DRAMPMEMDataPlacementType::kTiered.as_cache_data_placement(), + DRAMPMEMDataPlacementType::Tiered.as_cache_data_placement(), CacheDataPlacement::Tiered ); for ty in [ - DRAMPMEMDataPlacementType::kSideBySide, - DRAMPMEMDataPlacementType::kTiered, - DRAMPMEMDataPlacementType::kMaxCode, + DRAMPMEMDataPlacementType::SideBySide, + DRAMPMEMDataPlacementType::Tiered, + DRAMPMEMDataPlacementType::MaxCode, ] { - assert!(!ty.as_reference_name().is_empty()); + assert!(!ty.as_config_name().is_empty()); } } @@ -7483,8 +7621,8 @@ mod tests { cache.put_memory_only(k("mem"), b"m".to_vec()); assert_eq!(cache.get_memory(&k("mem")), Some(b"m".to_vec())); - assert!(cache.get_capacity(CacheInstanceType::kDRAM) > 0); - let _ = cache.get_used(CacheInstanceType::kDRAM); + assert!(cache.get_capacity(CacheInstanceType::Dram) > 0); + let _ = cache.get_used(CacheInstanceType::Dram); } #[test] @@ -7510,8 +7648,8 @@ mod tests { // introspection is callable let _ = cache.used_space_for_tier(CacheTier::Memory); - let _ = cache.get_used(CacheInstanceType::kDRAM); - let _ = cache.get_replacement_policy_type(CacheInstanceType::kDRAM); + let _ = cache.get_used(CacheInstanceType::Dram); + let _ = cache.get_replacement_policy_type(CacheInstanceType::Dram); let _ = cache.replacement_policy_for_tier(CacheTier::Memory); cache.reset().unwrap(); @@ -7553,16 +7691,16 @@ mod tests { mgr.SetWriteEnabled(false); assert!(!mgr.WriteEnabled()); mgr.Start(); - let _ = mgr.put(("k1", b"v1".to_vec()), WriteBufferType::kUserDataBuf); - let _ = mgr.put(("k2", b"v2".to_vec()), WriteBufferType::kUserDataBuf); - let _ = mgr.buffered_count(WriteBufferType::kUserDataBuf); + let _ = mgr.put(("k1", b"v1".to_vec()), WriteBufferType::UserDataBuf); + let _ = mgr.put(("k2", b"v2".to_vec()), WriteBufferType::UserDataBuf); + let _ = mgr.buffered_count(WriteBufferType::UserDataBuf); let _ = mgr.FlushBuffers(); let _ = mgr.flushed_records(); mgr.Stop(); let _ = BufferManager::new().capacity_per_buf(); let mut mgr2 = BufferManager::with_config(1024, 0.5, 512); - let mut wb = WriteBuffer::new(WriteBufferType::kUserDataBuf, 1024); + let mut wb = WriteBuffer::new(WriteBufferType::UserDataBuf, 1024); wb.PushBack("x", b"y".to_vec()); assert_eq!(mgr2.flush_buffer(wb), 1); } @@ -7631,12 +7769,12 @@ mod tests { assert_eq!(decode_colored_ptr(mask_colored_ptr_size(0, 3)).0, 3); let _ = MaskColoredPtrLBA(0, 4); let _ = MaskColoredPtrMemoryAddress(0, 0x10); - let _ = MaskColoredPtrRecordState(0, RecordStateType::kNormal); - let _ = mask_colored_ptr_record_state(0, RecordStateType::kPinned); + let _ = MaskColoredPtrRecordState(0, RecordStateType::Normal); + let _ = mask_colored_ptr_record_state(0, RecordStateType::Pinned); // BufferEncoder size calculators let encoder = BufferEncoder::new(4096); - let mut wb = WriteBuffer::new(WriteBufferType::kUserDataBuf, 1024); + let mut wb = WriteBuffer::new(WriteBufferType::UserDataBuf, 1024); wb.PushBack("k", b"v".to_vec()); let _ = encoder.calculate_encoded_data_size(&wb); let _ = encoder.calculate_encoded_oplog_size(&wb); @@ -7708,8 +7846,8 @@ mod tests { #[test] fn rdma_storage_engine_ops() { - let mut engine = RdmaStorageEngine::new(RdmaStorageEngineType::DRAM, 1 << 20); - assert!(matches!(engine.storage_type(), RdmaStorageEngineType::DRAM)); + let mut engine = RdmaStorageEngine::new(RdmaStorageEngineType::Dram, 1 << 20); + assert!(matches!(engine.storage_type(), RdmaStorageEngineType::Dram)); assert_eq!(engine.capacity(), 1 << 20); assert_eq!(engine.used(), 0); let ptr = engine.Put(b"key", b"value").expect("rdma put allocates"); @@ -7811,7 +7949,7 @@ mod tests { let _ = MatrixCacheBuilder::build_concurrent_simple_lru_cache(1024); let _ = MatrixCacheBuilder::build_memcached_wrapper(1024); - // DRAM-only options avoid needing an on-disk SSD tier + // Dram-only options avoid needing an on-disk Ssd tier let opts = || CacheOptions::new(1 << 16, 0, 0); let cache = MatrixCacheBuilder::build_cache(opts()); cache.put(CacheKey::string(0, "a"), b"1".to_vec()).unwrap(); @@ -7870,8 +8008,8 @@ mod tests { cache.release(h); } - cache.set_capacity_for_instance(CacheInstanceType::kDRAM, 2 << 20); - cache.set_replacement_policy_type(CacheInstanceType::kDRAM, CacheReplacementPolicy::Fifo); + cache.set_capacity_for_instance(CacheInstanceType::Dram, 2 << 20); + cache.set_replacement_policy_type(CacheInstanceType::Dram, CacheReplacementPolicy::Fifo); let policy = cache.production_tiering_policy(); cache.update_production_tiering_policy(policy); } @@ -7890,7 +8028,7 @@ mod tests { let mut table = RdmaHashTable::>::new(16); let key = b"hkey".to_vec(); assert!(table.Get(&key).addr.is_none()); - let _ = table.Put(key.clone(), 0x1000, 5, RdmaStorageEngineType::DRAM); + let _ = table.Put(key.clone(), 0x1000, 5, RdmaStorageEngineType::Dram); let _ = table.Get(&key); let _ = table.Del(&key); let _ = table.get_bucket(0); @@ -7898,14 +8036,14 @@ mod tests { } #[test] - fn parity_alloc_utils_parse_allocate_persist_thread_ids_and_pmem_files() { - assert_eq!(ParseAllocatorType("Log"), AllocatorType::kLogBasedAllocator); + fn alloc_utils_parse_allocate_persist_thread_ids_and_pmem_files() { + assert_eq!(ParseAllocatorType("Log"), AllocatorType::LogBasedAllocator); assert_eq!( ParseAllocatorType("Pool"), - AllocatorType::kPoolBasedAllocator + AllocatorType::PoolBasedAllocator ); - assert_eq!(ParseAllocatorType("Jemalloc"), AllocatorType::kJeAllocator); - assert_eq!(ParseAllocatorType("missing"), AllocatorType::kMaxCode); + assert_eq!(ParseAllocatorType("Jemalloc"), AllocatorType::JeAllocator); + assert_eq!(ParseAllocatorType("missing"), AllocatorType::MaxCode); let ptr = DramAllocateObject(4096, 4096).unwrap(); assert_eq!(ptr % 4096, 0); @@ -7951,7 +8089,7 @@ mod tests { } #[test] - fn parity_simple_log_based_allocator_allocates_seals_frees_and_reports_stats() { + fn simple_log_based_allocator_allocates_seals_frees_and_reports_stats() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(16); let ptr = allocator.Allocate(8).unwrap(); @@ -7997,7 +8135,7 @@ mod tests { } #[test] - fn parity_simple_log_based_allocator_supports_trait_consumers() { + fn simple_log_based_allocator_supports_trait_consumers() { fn allocate_and_seal( allocator: &mut A, ) -> Result { @@ -8019,7 +8157,7 @@ mod tests { } #[test] - fn parity_storage_gc_controller_lifecycle_pause_and_force_gc_match_surface() { + fn storage_gc_controller_lifecycle_pause_and_force_gc_match_surface() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); let ptr = allocator.Allocate(8).unwrap(); allocator.write(ptr, b"gc-ready").unwrap(); @@ -8050,7 +8188,7 @@ mod tests { } #[test] - fn parity_storage_gc_controller_poll_paces_collection_checks() { + fn storage_gc_controller_poll_paces_collection_checks() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); let first = allocator.Allocate(8).unwrap(); allocator.Free(first, 8).unwrap(); @@ -8094,7 +8232,7 @@ mod tests { } #[test] - fn parity_storage_gc_controller_respects_enable_gate_and_manual_gc_job() { + fn storage_gc_controller_respects_enable_gate_and_manual_gc_job() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); let ptr_a = allocator.Allocate(4).unwrap(); let ptr_b = allocator.Allocate(4).unwrap(); @@ -8118,7 +8256,7 @@ mod tests { } #[test] - fn parity_cache_executor_reuses_common_and_gc_executors_and_runs_tasks() { + fn cache_executor_reuses_common_and_gc_executors_and_runs_tasks() { CacheExecutor::DestroyAllExecutors(); CacheExecutor::Configure(CacheExecutorConfig { common_executor_num_threads: 3, @@ -8148,7 +8286,7 @@ mod tests { } #[test] - fn parity_cache_executor_creates_pmem_numa_executors_and_destroy_resets() { + fn cache_executor_creates_pmem_numa_executors_and_destroy_resets() { CacheExecutor::DestroyAllExecutors(); CacheExecutor::Configure(CacheExecutorConfig { common_executor_num_threads: 1, @@ -8178,7 +8316,7 @@ mod tests { } #[test] - fn parity_async_writer_runs_write_then_callback_and_tracks_counters() { + fn async_writer_runs_write_then_callback_and_tracks_counters() { let allocator = SimpleLogBasedMemoryAllocator::with_capacity(128); let mut writer = AsyncWriter::new(allocator); @@ -8208,7 +8346,7 @@ mod tests { } #[test] - fn parity_async_writer_preserves_addr_and_stop_rejects_new_writes() { + fn async_writer_preserves_addr_and_stop_rejects_new_writes() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(128); let existing = allocator.Allocate(4).unwrap(); allocator.write(existing, b"seed").unwrap(); @@ -8251,13 +8389,13 @@ mod tests { } #[test] - fn parity_pmem_dispatcher_round_robins_put_tasks_across_numa_writers() { + fn pmem_dispatcher_round_robins_put_tasks_across_numa_writers() { let mut dispatcher = PMemDispatcher::new(2, 128); assert!(dispatcher.Start()); assert_eq!(dispatcher.numa_count(), 2); assert_eq!( dispatcher.allocator_type(), - AllocatorType::kLogBasedAllocator + AllocatorType::LogBasedAllocator ); for (key, value) in [("first", b"one".to_vec()), ("second", b"two".to_vec())] { @@ -8292,7 +8430,7 @@ mod tests { } #[test] - fn parity_pmem_dispatcher_routes_addr_tasks_to_owner_numa() { + fn pmem_dispatcher_routes_addr_tasks_to_owner_numa() { let mut alloc0 = SimpleLogBasedMemoryAllocator::with_capacity_and_base(128, 1 << 48); let mut alloc1 = SimpleLogBasedMemoryAllocator::with_capacity_and_base(128, 2 << 48); let ptr0 = alloc0.Allocate(4).unwrap(); @@ -8327,7 +8465,7 @@ mod tests { } #[test] - fn parity_pmem_dispatcher_supports_test_allocator_access_and_stop() { + fn pmem_dispatcher_supports_test_allocator_access_and_stop() { let mut dispatcher = PMemDispatcher::new(2, 128); dispatcher.Start(); @@ -8368,7 +8506,51 @@ mod tests { } #[test] - fn parity_replacement_fifo_evicts_oldest_and_invokes_handler() { + fn replacement_policies_stay_usable_after_reset() { + // A successful reset empties the index; it does not retire the + // policy. Asserting only that the post-reset put reports no + // evictions would not catch a regression here, because a policy + // that silently discards the buffer reports no evictions too. + // Check the buffer actually landed. + let mut fifo = ReplacementFIFO::new(1 << 20); + fifo.Init().unwrap(); + fifo.Put(test_buffer("before", b"1")); + assert!(fifo.GetUsedSpace() > 0); + + fifo.Reset().unwrap(); + assert_eq!(fifo.GetUsedSpace(), 0); + assert_eq!(fifo.GetItemNum(), 0); + assert!(fifo.Get("before").is_none()); + + assert!(fifo.Put(test_buffer("after", b"2")).is_empty()); + assert_eq!( + fifo.Peek("after").map(|buffer| buffer.Data().to_vec()), + Some(b"2".to_vec()), + "a reset fifo must still accept buffers" + ); + assert!(fifo.GetUsedSpace() > 0); + + let mut slru = ReplacementSLRU::new(1 << 20); + slru.Init().unwrap(); + slru.Put(test_buffer("before", b"1")); + assert!(slru.GetUsedSpace() > 0); + + slru.Reset().unwrap(); + assert_eq!(slru.GetUsedSpace(), 0); + assert_eq!(slru.GetItemNum(), 0); + assert!(slru.Get("before").is_none()); + + assert!(slru.Put(test_buffer("after", b"2")).is_empty()); + assert_eq!( + slru.Peek("after").map(|buffer| buffer.Data().to_vec()), + Some(b"2".to_vec()), + "a reset slru must still accept buffers" + ); + assert!(slru.GetUsedSpace() > 0); + } + + #[test] + fn replacement_fifo_evicts_oldest_and_invokes_handler() { let mut fifo = ReplacementFIFO::new(5); fifo.Init().unwrap(); let evicted_keys = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -8393,7 +8575,7 @@ mod tests { } #[test] - fn parity_replacement_fifo_update_guards_raw_data_and_preserves_order() { + fn replacement_fifo_update_guards_raw_data_and_preserves_order() { let mut fifo = ReplacementFIFO::new(32); fifo.Init().unwrap(); fifo.Put(test_buffer("guarded", b"old")); @@ -8413,7 +8595,7 @@ mod tests { } #[test] - fn parity_replacement_fifo_overwrite_keeps_original_queue_position() { + fn replacement_fifo_overwrite_keeps_original_queue_position() { let mut fifo = ReplacementFIFO::new(6); fifo.Init().unwrap(); // Three 2-byte entries exactly fill the policy. @@ -8440,7 +8622,7 @@ mod tests { } #[test] - fn parity_replacement_fifo_delete_leaves_no_queue_tombstone() { + fn replacement_fifo_delete_leaves_no_queue_tombstone() { let mut fifo = ReplacementFIFO::new(1 << 12); fifo.Init().unwrap(); for index in 0..256 { @@ -8466,7 +8648,7 @@ mod tests { } #[test] - fn parity_replacement_slru_get_records_access_without_reordering() { + fn replacement_slru_get_records_access_without_reordering() { let mut slru = ReplacementSLRU::with_num_segments(100, 1); slru.Init().unwrap(); slru.TEST_ConfigLRUMaintainer(false); @@ -8532,7 +8714,7 @@ mod tests { } #[test] - fn parity_replacement_slru_tracks_hot_warm_cold_and_fetch_flags() { + fn replacement_slru_tracks_hot_warm_cold_and_fetch_flags() { let mut slru = ReplacementSLRU::new(6); slru.Init().unwrap(); slru.TEST_ConfigLRUMaintainer(false); @@ -8555,7 +8737,7 @@ mod tests { } #[test] - fn parity_replacement_slru_update_delete_and_capacity_shrink() { + fn replacement_slru_update_delete_and_capacity_shrink() { let mut slru = ReplacementSLRU::new(32); slru.Init().unwrap(); slru.Put(test_buffer("x", b"old")); @@ -8577,7 +8759,7 @@ mod tests { } #[test] - fn parity_replacement_slru_resolves_segment_count_from_capacity_and_request() { + fn replacement_slru_resolves_segment_count_from_capacity_and_request() { // A capacity smaller than the segment count collapses to one segment, // otherwise every segment would get a zero byte budget. assert_eq!(ReplacementSLRU::new(6).num_segments(), 1); @@ -8603,7 +8785,7 @@ mod tests { } #[test] - fn parity_replacement_slru_shards_keys_and_bounds_each_segment() { + fn replacement_slru_shards_keys_and_bounds_each_segment() { let mut slru = ReplacementSLRU::new(1 << 16); slru.Init().unwrap(); assert_eq!(slru.num_segments(), SLRU_DEFAULT_NUM_SEGMENTS); @@ -8647,7 +8829,7 @@ mod tests { } #[test] - fn parity_replacement_slru_accounting_survives_overwrite_delete_and_reuse() { + fn replacement_slru_accounting_survives_overwrite_delete_and_reuse() { let mut slru = ReplacementSLRU::with_num_segments(1 << 14, 4); slru.Init().unwrap(); @@ -8690,7 +8872,7 @@ mod tests { } #[test] - fn parity_replacement_slru_maintainer_promotes_active_and_demotes_untouched() { + fn replacement_slru_maintainer_promotes_active_and_demotes_untouched() { let mut slru = ReplacementSLRU::with_num_segments(100, 1); slru.Init().unwrap(); slru.TEST_ConfigLRUMaintainer(false); @@ -8744,7 +8926,7 @@ mod tests { } #[test] - fn parity_replacement_slru_maintainer_evicts_cold_tail_over_segment_budget() { + fn replacement_slru_maintainer_evicts_cold_tail_over_segment_budget() { let mut slru = ReplacementSLRU::with_num_segments(40, 1); slru.Init().unwrap(); // Give hot and warm no share of the budget, so every insert is demoted @@ -8786,7 +8968,7 @@ mod tests { } #[test] - fn parity_concurrent_slru_matches_the_single_threaded_segment_layout() { + fn concurrent_slru_matches_the_single_threaded_segment_layout() { let concurrent = ConcurrentReplacementSLRU::with_num_segments(1 << 16, 256); let single = ReplacementSLRU::with_num_segments(1 << 16, 256); assert_eq!(concurrent.num_segments(), single.num_segments()); @@ -8805,7 +8987,7 @@ mod tests { } #[test] - fn parity_concurrent_slru_serves_threads_through_per_segment_locks() { + fn concurrent_slru_serves_threads_through_per_segment_locks() { let policy = ConcurrentReplacementSLRU::with_num_segments(1 << 16, 64); policy.Init().unwrap(); @@ -8844,7 +9026,7 @@ mod tests { } #[test] - fn parity_concurrent_slru_reports_evictions_from_every_segment() { + fn concurrent_slru_reports_evictions_from_every_segment() { let policy = ConcurrentReplacementSLRU::with_num_segments(256, 4); policy.Init().unwrap(); let evicted = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -8867,7 +9049,7 @@ mod tests { } #[test] - fn parity_concurrent_slru_round_trips_values_and_maintainer_passes() { + fn concurrent_slru_round_trips_values_and_maintainer_passes() { let policy = ConcurrentReplacementSLRU::with_num_segments(1 << 14, 8); policy.Init().unwrap(); policy.Put(test_buffer("round-trip", b"value")); @@ -8919,7 +9101,7 @@ mod tests { // A hit moves the key to the back and never duplicates it, which is // what a plain deque needs a full rescan to guarantee. - assert!(order.touch(&order_key(0))); + assert!(order.move_to_back(&order_key(0))); assert_eq!(order.back(), Some(&order_key(0))); assert_eq!(order.len(), 4); assert_eq!( @@ -8928,14 +9110,14 @@ mod tests { ); // Touching the key that is already most recent is a no-op. - assert!(order.touch(&order_key(0))); + assert!(order.move_to_back(&order_key(0))); assert_eq!( order_keys(&order), vec![order_key(1), order_key(2), order_key(3), order_key(0)] ); // Touching an absent key reports it and changes nothing. - assert!(!order.touch(&order_key(99))); + assert!(!order.move_to_back(&order_key(99))); assert_eq!(order.len(), 4); // Eviction takes the least recently used first. @@ -8986,6 +9168,44 @@ mod tests { assert_eq!(empty.iter_rev().count(), 0); } + #[test] + fn zero_copy_lru_keeps_counting_removed_but_pinned_bytes() { + let cache = ZeroCopySimpleLRUCache::new(4 * 64); + let key = CacheKey::string(0, "pinned-entry"); + let handle = cache + .InsertPinned(key.clone(), vec![118u8; 32], 64) + .unwrap() + .expect("pinned handle"); + assert_eq!(cache.Size(), 64); + + // The entry leaves the index, but a handle still holds the value, so + // those bytes are still resident and must keep counting. Releasing + // them here would let the cache admit data it has no room for. + cache.Remove(&key).unwrap(); + assert!(cache.Lookup(&key).unwrap().is_none()); + assert_eq!( + cache.Size(), + 64, + "a removed entry that is still pinned must stay accounted" + ); + + // Dropping the last pin is what actually frees the space. + cache.Release(handle); + assert_eq!(cache.Size(), 0); + } + + #[test] + fn zero_copy_lru_frees_removed_bytes_when_no_handle_holds_them() { + let cache = ZeroCopySimpleLRUCache::new(4 * 64); + let key = CacheKey::string(0, "unpinned-entry"); + cache.Insert(key.clone(), vec![118u8; 32], 64).unwrap(); + assert_eq!(cache.Size(), 64); + + // Nothing holds this one, so removal frees it immediately. + cache.Remove(&key).unwrap(); + assert_eq!(cache.Size(), 0); + } + #[test] fn simple_lru_evicts_the_coldest_entry_first() { let cache = SimpleLRUCache::new(3 * 64); @@ -9052,9 +9272,9 @@ mod tests { deque.retain(|candidate| candidate != &key); deque.push_back(key.clone()); } - assert!(order.touch(&key)); + assert!(order.move_to_back(&key)); } else { - assert!(!order.touch(&key)); + assert!(!order.move_to_back(&key)); } } 3 => { @@ -9096,7 +9316,7 @@ mod tests { } #[test] - fn parity_hash_uint64_matches_matrixcache_vectors() { + fn hash_uint64_matches_matrixcache_vectors() { assert_eq!(hash_uint64(0), 0x5b03_af84_387a_42c6); assert_eq!(hash_uint64(1), 0xa13a_3e40_1240_2345); assert_eq!(hash_uint64(2), 0xcd41_43fa_e38a_71fe); @@ -9106,7 +9326,7 @@ mod tests { } #[test] - fn parity_murmur_hash2_matches_matrixcache_vectors() { + fn murmur_hash2_matches_matrixcache_vectors() { assert_eq!(mur_mur_hash2(b""), 0xca88_1466); assert_eq!(mur_mur_hash2(b"a"), 0xe94e_6ebd); assert_eq!(mur_mur_hash2(b"abc"), 0x6d5e_3568); @@ -9131,7 +9351,7 @@ mod tests { } #[test] - fn parity_tools_utils_random_and_hashed_key_helpers_match_surface() { + fn tools_utils_random_and_hashed_key_helpers_match_surface() { assert_eq!(xxh32_with_seed(b"", 0), 0x02cc_5d05); assert_eq!(xxh32_with_seed(b"hello", 0), 0xfb00_77f9); @@ -9157,7 +9377,7 @@ mod tests { } #[test] - fn parity_round_up_matches_align_util_macro_semantics() { + fn round_up_matches_align_util_macro_semantics() { assert_eq!(round_up(0, 8), 0); assert_eq!(round_up(1, 8), 8); assert_eq!(round_up(8, 8), 8); @@ -9168,7 +9388,7 @@ mod tests { } #[test] - fn parity_numa_info_exposes_stable_single_node_topology() { + fn numa_info_exposes_stable_single_node_topology() { NumaInfo::Init(); assert!(NumaInfo::GetNumAllCores() >= 1); assert!(NumaInfo::GetNumOnlineCores() >= NumaInfo::GetNumAllCores()); @@ -9190,8 +9410,8 @@ mod tests { assert_eq!(cache.shard_count(), 4); assert_eq!(cache.capacity_for_tier(CacheTier::Memory), 96); assert_eq!(cache.CapacityForTier(CacheTier::Ssd), 4096); - assert_eq!(cache.GetCapacity(CacheInstanceType::kDRAM), 96); - assert_eq!(cache.GetCapacity(CacheInstanceType::kSSD), 4096); + assert_eq!(cache.GetCapacity(CacheInstanceType::Dram), 96); + assert_eq!(cache.GetCapacity(CacheInstanceType::Ssd), 4096); let config_cache = MatrixCacheBuilder::build_sharded_cache(CacheOptions::new(32, 32, 0), 2); assert!(config_cache.stop()); config_cache @@ -9231,8 +9451,8 @@ mod tests { assert!(latency.put_count >= keys.len() as u64); assert!(latency.get_count >= keys.len() as u64); assert!(latency.histogram_ready); - assert!(cache.GetUsed(CacheInstanceType::kDRAM) > 0); - assert!(cache.GetUsed(CacheInstanceType::kSSD) > 0); + assert!(cache.GetUsed(CacheInstanceType::Dram) > 0); + assert!(cache.GetUsed(CacheInstanceType::Ssd) > 0); let repeated = keys[3].clone(); let other = keys[7].clone(); @@ -9262,9 +9482,9 @@ mod tests { cache.ReplacementPolicyForTier(CacheTier::Memory), CacheReplacementPolicy::WeightedHotnessLru ); - cache.SetReplacementPolicyType(CacheInstanceType::kSSD, CacheReplacementPolicy::Fifo); + cache.SetReplacementPolicyType(CacheInstanceType::Ssd, CacheReplacementPolicy::Fifo); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kSSD), + cache.GetReplacementPolicyType(CacheInstanceType::Ssd), CacheReplacementPolicy::Fifo ); assert_eq!( @@ -9279,33 +9499,33 @@ mod tests { let eviction = cache.EvictionReport(); assert!(eviction.memory_capacity_evictions > 0); assert!(eviction.memory_slot_evictions > 0); - cache.SetCapacityForInstance(CacheInstanceType::kSSD, 1024); - assert_eq!(cache.GetCapacity(CacheInstanceType::kSSD), 1024); + cache.SetCapacityForInstance(CacheInstanceType::Ssd, 1024); + assert_eq!(cache.GetCapacity(CacheInstanceType::Ssd), 1024); cache.SetReplacementPolicyForTier(CacheTier::Memory, CacheReplacementPolicy::Fifo); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::Fifo ); let running_cache = MatrixCacheBuilder::build_sharded_cache(CacheOptions::new(32, 0, 0), 2); running_cache.start().unwrap(); assert!(matches!( running_cache.TrySetReplacementPolicyType( - CacheInstanceType::kDRAM, + CacheInstanceType::Dram, CacheReplacementPolicy::Fifo, ), Err(CacheError::AlreadyStarted) )); assert_eq!( - running_cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + running_cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::WeightedHotnessLru ); assert!(matches!( cache.TrySetReplacementPolicyType( - CacheInstanceType::kUnified, + CacheInstanceType::Unified, CacheReplacementPolicy::Fifo, ), - Err(CacheError::UnsupportedInstance(CacheInstanceType::kUnified)) + Err(CacheError::UnsupportedInstance(CacheInstanceType::Unified)) )); assert!(matches!( cache.TrySetReplacementPolicyForTier(CacheTier::Reject, CacheReplacementPolicy::Fifo), @@ -9318,11 +9538,11 @@ mod tests { 2, ); assert_eq!( - api.capacity_for_instance_cache(CacheInstanceType::kDRAM), + api.capacity_for_instance_cache(CacheInstanceType::Dram), 64 ); assert_eq!( - api.capacity_for_instance_cache(CacheInstanceType::kSSD), + api.capacity_for_instance_cache(CacheInstanceType::Ssd), 1024 ); let trait_key = CacheKey::string(7, "trait-key"); @@ -9332,9 +9552,9 @@ mod tests { api.lookup_cache(&trait_key).unwrap().unwrap(), b"trait-value".to_vec() ); - assert!(api.used_cache(CacheInstanceType::kDRAM) > 0); - api.set_capacity_for_instance_cache(CacheInstanceType::kDRAM, 8); - assert_eq!(api.capacity_for_instance_cache(CacheInstanceType::kDRAM), 8); + assert!(api.used_cache(CacheInstanceType::Dram) > 0); + api.set_capacity_for_instance_cache(CacheInstanceType::Dram, 8); + assert_eq!(api.capacity_for_instance_cache(CacheInstanceType::Dram), 8); api.reset_cache().unwrap(); assert_eq!(api.lookup_cache(&trait_key).unwrap(), None); @@ -9504,7 +9724,7 @@ mod tests { cache .test_insert( - CacheInstanceType::kPMEM, + CacheInstanceType::Pmem, acquired.clone(), b"pmem".to_vec(), 4, From 99c07757da9172289494a2ad9b1b253f59c9421f Mon Sep 17 00:00:00 2001 From: bjmeetsfo Date: Wed, 26 Aug 2026 18:15:58 -0700 Subject: [PATCH 2/3] say is_multiple_of in the hex decoder Keeps this branch clean under the Rust version the crate is about to declare. --- src/runtime/builder_and_gc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/builder_and_gc.rs b/src/runtime/builder_and_gc.rs index 1fe4246..d2b2f21 100644 --- a/src/runtime/builder_and_gc.rs +++ b/src/runtime/builder_and_gc.rs @@ -1889,7 +1889,7 @@ fn encode_manifest_field(value: &str) -> String { } fn decode_manifest_field(value: &str) -> Option { - if value.len() % 2 != 0 { + if !value.len().is_multiple_of(2) { return None; } let mut bytes = Vec::with_capacity(value.len() / 2); From 3820bb22718da7be9cd9da6ca5ee30f9f98ae8c0 Mon Sep 17 00:00:00 2001 From: bjmeetsfo Date: Wed, 26 Aug 2026 18:47:56 -0700 Subject: [PATCH 3/3] say as_chunks where the chunk size is a constant --- src/core/rdma.rs | 7 +++---- src/runtime/builder_and_gc.rs | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/core/rdma.rs b/src/core/rdma.rs index c98d579..5900d57 100644 --- a/src/core/rdma.rs +++ b/src/core/rdma.rs @@ -1662,9 +1662,9 @@ pub fn mur_mur_hash2_with_seed(key: &[u8], seed: u32) -> u32 { const R: u32 = 24; let mut h = seed ^ key.len() as u32; - let mut chunks = key.chunks_exact(4); - for chunk in &mut chunks { - let mut k = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + let (blocks, tail) = key.as_chunks::<4>(); + for chunk in blocks { + let mut k = u32::from_le_bytes(*chunk); k = k.wrapping_mul(M); k ^= k >> R; k = k.wrapping_mul(M); @@ -1673,7 +1673,6 @@ pub fn mur_mur_hash2_with_seed(key: &[u8], seed: u32) -> u32 { h ^= k; } - let tail = chunks.remainder(); match tail.len() { 3 => { h ^= ((tail[2] as i8 as i32) << 16) as u32; diff --git a/src/runtime/builder_and_gc.rs b/src/runtime/builder_and_gc.rs index d2b2f21..f567cb8 100644 --- a/src/runtime/builder_and_gc.rs +++ b/src/runtime/builder_and_gc.rs @@ -1893,7 +1893,7 @@ fn decode_manifest_field(value: &str) -> Option { return None; } let mut bytes = Vec::with_capacity(value.len() / 2); - for chunk in value.as_bytes().chunks_exact(2) { + for chunk in value.as_bytes().as_chunks::<2>().0 { let high = decode_hex_nibble(chunk[0])?; let low = decode_hex_nibble(chunk[1])?; bytes.push((high << 4) | low);