diff --git a/examples/cache_scaling_bench.rs b/examples/cache_scaling_bench.rs new file mode 100755 index 0000000..ba486a8 --- /dev/null +++ b/examples/cache_scaling_bench.rs @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 MatrixArkAI + +//! Read-path cost and read concurrency for the multi-tier cache. +//! +//! Two things are worth watching on the read path, and they move +//! independently. +//! +//! The first is what a single memory-tier hit costs, and whether that cost +//! stays put as the resident set grows. A lookup that scans anything will show +//! up here as a number that climbs with the entry count. +//! +//! The second is what happens when several threads read at once. A cache that +//! serialises its readers gets *slower* per operation as threads are added +//! rather than faster in aggregate, and that shows up as throughput falling in +//! the scaling table below. Reads currently take the cache lock exclusively +//! because the hit updates statistics, hotness and latency histograms on the +//! way out, so this table is the measurement to beat when that changes. +//! +//! The third table puts `MultiLayerCache` next to `ShardedMultiLayerCache` +//! holding the same total capacity. The sharded cache spreads keys over +//! independent shards, so readers of different shards do not queue behind one +//! another. It is the supported answer to the contention in the second table, +//! and this comparison is what says whether it earns its extra bookkeeping. +//! +//! ```text +//! cargo run --release --example cache_scaling_bench +//! cargo run --release --example cache_scaling_bench -- 8192 +//! ``` + +use matrixcache::{CacheKey, CacheOptions, MultiLayerCache, ShardedMultiLayerCache}; +use std::time::{Duration, Instant}; + +const VALUE_BYTES: usize = 64; +const REPEATS: usize = 5; + +fn bench_dir(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("matrixcache-scaling-{name}")) +} + +/// Keys plus a visit order that touches each one but not in insertion order. +fn workload(entries: usize) -> Vec { + (0..entries) + .map(|index| CacheKey::string(0, &format!("scaling-key-{index:010}"))) + .collect() +} + +fn scattered(index: usize, len: usize) -> usize { + index.wrapping_mul(2_654_435_761) % len.max(1) +} + +fn ns_per_op(elapsed: Duration, ops: usize) -> f64 { + if ops == 0 { + return 0.0; + } + elapsed.as_nanos() as f64 / ops as f64 +} + +/// Cost of a single-threaded memory-tier hit at a given resident entry count. +fn hit_cost(entries: usize) -> f64 { + let dir = bench_dir(&format!("hit-{entries}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::new(entries * 256, &dir); + cache.start().expect("start cache"); + let keys = workload(entries); + let value = vec![b'v'; VALUE_BYTES]; + for key in &keys { + cache.put(key.clone(), value.clone()).expect("put"); + } + + let ops = entries * 8; + let mut best = f64::MAX; + for _ in 0..REPEATS { + let started = Instant::now(); + for index in 0..ops { + let _ = cache.get(&keys[scattered(index, entries)]).expect("get"); + } + best = best.min(ns_per_op(started.elapsed(), ops)); + } + let _ = std::fs::remove_dir_all(&dir); + best +} + +/// Aggregate read throughput with `threads` readers sharing one cache. +fn read_throughput(entries: usize, threads: usize) -> f64 { + let dir = bench_dir(&format!("conc-{threads}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::new(entries * 256, &dir); + cache.start().expect("start cache"); + let keys = workload(entries); + let value = vec![b'v'; VALUE_BYTES]; + for key in &keys { + cache.put(key.clone(), value.clone()).expect("put"); + } + + let per_thread = 40_000usize; + let mut best = f64::MAX; + for _ in 0..3 { + let started = Instant::now(); + std::thread::scope(|scope| { + for thread in 0..threads { + let cache = &cache; + let keys = &keys; + scope.spawn(move || { + for index in 0..per_thread { + // Offset per thread so readers are not in lockstep. + let slot = scattered(index + thread * 7919, keys.len()); + let _ = cache.get(&keys[slot]).expect("get"); + } + }); + } + }); + best = best.min(ns_per_op(started.elapsed(), per_thread * threads)); + } + let _ = std::fs::remove_dir_all(&dir); + best +} + +/// Options shared by both cache shapes, so the comparison is like for like. +fn scaling_options(dir: &std::path::Path, entries: usize) -> CacheOptions { + CacheOptions::new(entries * 256, 0, 0).with_ssd_paths(vec![dir.to_path_buf()]) +} + +/// Read throughput for the single-lock cache, built from shared options. +fn single_lock_throughput(entries: usize, threads: usize) -> f64 { + let dir = bench_dir(&format!("single-{threads}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::with_options(scaling_options(&dir, entries)); + cache.start().expect("start cache"); + let keys = workload(entries); + let value = vec![b'v'; VALUE_BYTES]; + for key in &keys { + cache.put(key.clone(), value.clone()).expect("put"); + } + let best = drive_readers(threads, &keys, |key| { + let _ = cache.get(key).expect("get"); + }); + let _ = std::fs::remove_dir_all(&dir); + best +} + +/// Read throughput for the sharded cache holding the same total capacity. +fn sharded_throughput(entries: usize, threads: usize, shards: usize) -> f64 { + let dir = bench_dir(&format!("sharded-{shards}-{threads}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = ShardedMultiLayerCache::with_options(scaling_options(&dir, entries), shards); + cache.start().expect("start cache"); + let keys = workload(entries); + let value = vec![b'v'; VALUE_BYTES]; + for key in &keys { + cache.put(key.clone(), value.clone()).expect("put"); + } + let best = drive_readers(threads, &keys, |key| { + let _ = cache.get(key).expect("get"); + }); + let _ = std::fs::remove_dir_all(&dir); + best +} + +/// Run `threads` readers over `keys` and return the best ns/op of three runs. +fn drive_readers(threads: usize, keys: &[CacheKey], read: F) -> f64 +where + F: Fn(&CacheKey) + Sync, +{ + let per_thread = 40_000usize; + let mut best = f64::MAX; + for _ in 0..3 { + let started = Instant::now(); + std::thread::scope(|scope| { + for thread in 0..threads { + let read = &read; + scope.spawn(move || { + for index in 0..per_thread { + // Offset per thread so readers are not in lockstep. + let slot = scattered(index + thread * 7919, keys.len()); + read(&keys[slot]); + } + }); + } + }); + best = best.min(ns_per_op(started.elapsed(), per_thread * threads)); + } + best +} + +fn main() { + let max_entries: usize = std::env::args() + .nth(1) + .and_then(|value| value.parse().ok()) + .unwrap_or(4_096); + + // Warm the allocator before the first measured case. + let _ = hit_cost(256); + + println!("memory-tier hit, single thread"); + println!("{:>10} {:>14}", "entries", "ns/op"); + let mut size = 1_024usize; + while size <= max_entries { + println!("{size:>10} {:>14.1}", hit_cost(size)); + size *= 4; + } + + println!(); + println!("read throughput, {max_entries} resident entries"); + println!("{:>10} {:>14} {:>14}", "threads", "ns/op", "Mops/s"); + for &threads in &[1usize, 2, 4, 8] { + let ns = read_throughput(max_entries, threads); + let mops = if ns > 0.0 { 1_000.0 / ns } else { 0.0 }; + println!("{threads:>10} {ns:>14.1} {mops:>14.2}"); + } + + println!(); + println!("single lock vs sharded, same total capacity, Mops/s"); + println!( + "{:>10} {:>14} {:>14} {:>10}", + "threads", "single", "sharded", "speedup" + ); + for &threads in &[1usize, 2, 4, 8] { + let single_ns = single_lock_throughput(max_entries, threads); + let sharded_ns = sharded_throughput(max_entries, threads, 16); + let single = if single_ns > 0.0 { + 1_000.0 / single_ns + } else { + 0.0 + }; + let sharded = if sharded_ns > 0.0 { + 1_000.0 / sharded_ns + } else { + 0.0 + }; + let speedup = if single > 0.0 { sharded / single } else { 0.0 }; + println!("{threads:>10} {single:>14.2} {sharded:>14.2} {speedup:>9.2}x"); + } +} diff --git a/examples/eviction_bench.rs b/examples/eviction_bench.rs new file mode 100755 index 0000000..1b7fdcb --- /dev/null +++ b/examples/eviction_bench.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 MatrixArkAI + +//! Cost of choosing an eviction victim, as the resident set grows. +//! +//! A cache at capacity evicts on almost every write, so whatever victim +//! selection costs is paid per write for the life of the cache. The thing to +//! watch is whether that cost stays put as the cache fills: a selector that +//! inspects every resident entry shows up here as a per-write cost that climbs +//! with the entry count, while one that inspects a bounded number of +//! candidates shows up as a flat line. +//! +//! Two numbers are reported per size. The first is wall time per write, which +//! is what a caller feels. The second is the number of candidate groups the +//! selector formed per evicted entry, which the cache already counts; it is +//! immune to load on the machine and is the number that says whether the +//! algorithm changed or only the weather did. +//! +//! ```text +//! cargo run --release --no-default-features --example eviction_bench +//! ``` + +use matrixcache::{CacheKey, CacheOptions, MultiLayerCache}; +use std::time::Instant; + +const VALUE_BYTES: usize = 64; +/// Room for the value plus its per-entry overhead, so `entries` really fit. +const SLOT_BYTES: usize = VALUE_BYTES; + +fn bench_dir(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("matrixcache-eviction-{name}")) +} + +fn key(index: usize) -> CacheKey { + CacheKey::string(0, &format!("eviction-key-{index:010}")) +} + +/// Spread successive steps over `len` slots so reads are not in insertion +/// order, which would flatter any policy that evicts from one end. +fn scattered(index: usize, len: usize) -> usize { + index.wrapping_mul(2_654_435_761) % len.max(1) +} + +/// Fill to capacity, then keep writing so every write evicts. +fn steady_state(entries: usize) -> (f64, f64) { + let dir = bench_dir(&format!("steady-{entries}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::with_options( + CacheOptions::new(entries * SLOT_BYTES, 0, 0).with_ssd_paths(vec![dir.clone()]), + ); + cache.start().expect("start cache"); + + let value = vec![b'v'; VALUE_BYTES]; + // Fill to capacity. Keys are built up front so the timed region below + // measures the cache rather than key formatting. + for index in 0..entries { + cache.put(key(index), value.clone()).expect("put"); + } + + let writes = 2_000usize; + let fresh: Vec = (entries..entries + writes).map(key).collect(); + + let before = cache.stats(); + let started = Instant::now(); + for k in &fresh { + cache.put(k.clone(), value.clone()).expect("put"); + } + let elapsed = started.elapsed(); + let after = cache.stats(); + + let evictions = after + .memory_evictions + .saturating_sub(before.memory_evictions); + let groups = after + .eviction_sampled_groups + .saturating_sub(before.eviction_sampled_groups); + + let ns_per_write = elapsed.as_nanos() as f64 / writes as f64; + let groups_per_eviction = if evictions == 0 { + 0.0 + } else { + groups as f64 / evictions as f64 + }; + + cache.stop(); + let _ = std::fs::remove_dir_all(&dir); + (ns_per_write, groups_per_eviction) +} + +/// Hit rate under a skewed read-through workload. +/// +/// Bounding the candidate search only pays off if it still throws out the +/// right entries. This drives a working set several times larger than the +/// cache, with most reads landing on a small hot subset, and reports the share +/// of reads the cache served. A selector that evicts hot entries shows up here +/// as a hit rate below what the hot subset alone would guarantee. +fn hit_rate(entries: usize) -> f64 { + let dir = bench_dir(&format!("hitrate-{entries}")); + let _ = std::fs::remove_dir_all(&dir); + let cache = MultiLayerCache::with_options( + CacheOptions::new(entries * SLOT_BYTES, 0, 0).with_ssd_paths(vec![dir.clone()]), + ); + cache.start().expect("start cache"); + + let value = vec![b'v'; VALUE_BYTES]; + // Four times as many keys as fit, with a hot subset that is half the + // cache, so a selector that protects hot entries can hold all of them. + let universe = entries * 4; + let hot = entries / 2; + let keys: Vec = (0..universe).map(key).collect(); + + let reads = 400_000usize; + let mut hits = 0usize; + for step in 0..reads { + // Four reads in five land in the hot subset; the rest sweep the + // universe and are the pressure that forces eviction. + let slot = if step % 5 < 4 { + scattered(step, hot) + } else { + scattered(step, universe) + }; + let k = &keys[slot]; + if cache.get(k).expect("get").is_some() { + hits += 1; + } else { + cache.put(k.clone(), value.clone()).expect("put"); + } + } + + cache.stop(); + let _ = std::fs::remove_dir_all(&dir); + hits as f64 * 100.0 / reads as f64 +} + +fn main() { + println!("steady-state write cost with the cache at capacity"); + println!( + "{:>10} {:>14} {:>22}", + "entries", "ns/write", "groups/eviction" + ); + for entries in [1_024usize, 2_048, 4_096, 8_192, 16_384, 32_768] { + let (ns, groups) = steady_state(entries); + println!("{entries:>10} {ns:>14.0} {groups:>22.1}"); + } + + println!(); + println!("hit rate, working set 4x the cache, 80% of reads on a hot half-cache"); + println!("{:>10} {:>14}", "entries", "hit rate %"); + for entries in [1_024usize, 4_096, 16_384] { + println!("{entries:>10} {:>14.2}", hit_rate(entries)); + } +} diff --git a/examples/policy_bench.rs b/examples/policy_bench.rs index 71d93e2..62705c7 100755 --- a/examples/policy_bench.rs +++ b/examples/policy_bench.rs @@ -5,7 +5,7 @@ //! //! Every policy keeps its keys in intrusive doubly-linked lists over a shared //! node arena, so a lookup or a delete unlinks an entry in constant time rather -//! than rescanning a list of keys. The segmented LRU additionally partitions +//! than rescanning a list of keys. The segmented Lru additionally partitions //! its index and lists into segments, each with its own byte budget, so //! eviction only ever walks a segment-local list. //! diff --git a/examples/rocksdb_parity_bench.rs b/examples/rocksdb_backend_bench.rs similarity index 96% rename from examples/rocksdb_parity_bench.rs rename to examples/rocksdb_backend_bench.rs index 831183c..7f7543e 100644 --- a/examples/rocksdb_parity_bench.rs +++ b/examples/rocksdb_backend_bench.rs @@ -122,7 +122,7 @@ fn parse_config() -> BenchConfig { } "--help" | "-h" => { println!( - "usage: rocksdb_parity_bench [iterations] [--iterations N] \ + "usage: rocksdb_backend_bench [iterations] [--iterations N] \ [--value-bytes N] [--dram-capacity-bytes N] \ [--pmem-capacity-bytes N] [--ssd-capacity-bytes N] \ [--placement-threshold-bytes N] \ @@ -279,7 +279,7 @@ fn main() { && restart_disk_refill_ready; println!("{{"); - println!(" \"report_version\": \"matrixcache_rocksdb_parity_v1\","); + println!(" \"report_version\": \"matrixcache_rocksdb_backend_v1\","); println!( " \"backend\": \"{}\",", if cfg!(feature = "rocksdb-ssd") { @@ -367,28 +367,28 @@ fn main() { println!(" \"matrixcache_contract_evidence\": {{"); println!(" \"dram_to_pmem_eviction\": {{"); println!(" \"observed\": {},", dram_to_pmem_eviction); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"memory_evictions > 0 && pmem_fills > 0\","); println!(" \"memory_evictions\": {},", stats.memory_evictions); println!(" \"pmem_fills\": {}", stats.pmem_fills); println!(" }},"); println!(" \"pmem_to_ssd_eviction\": {{"); println!(" \"observed\": {},", pmem_to_ssd_eviction); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"pmem_evictions > 0 && disk_fills > 0\","); println!(" \"pmem_evictions\": {},", stats.pmem_evictions); println!(" \"disk_fills\": {}", stats.disk_fills); println!(" }},"); println!(" \"ssd_read_through_refill\": {{"); println!(" \"observed\": {},", ssd_read_through_refill); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"cold_ssd_refills > 0 && refill_failures == 0\","); println!(" \"cold_ssd_refills\": {},", cold_ssd_refills); println!(" \"refill_failures\": {}", stats.refill_failures); println!(" }},"); println!(" \"replacement_soak\": {{"); println!(" \"observed\": {},", replacement_soak_ready); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"replacement_policy_soak.passed\","); println!(" \"iterations\": {},", soak_iterations); println!(" \"reasons\": {:?}", soak.reasons); @@ -398,7 +398,7 @@ fn main() { " \"observed\": {},", async_writeback_backpressure_ready ); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"observed_async_writeback_backpressure > 0\","); println!( " \"observed_async_writeback_backpressure\": {}", @@ -407,7 +407,7 @@ fn main() { println!(" }},"); println!(" \"restart_disk_refill\": {{"); println!(" \"observed\": {},", restart_disk_refill_ready); - println!(" \"source\": \"matrixcache_rocksdb_parity_bench\","); + println!(" \"source\": \"matrixcache_rocksdb_backend_bench\","); println!(" \"metric\": \"replacement_policy_soak.restart_disk_refill_ready\","); println!( " \"restart_disk_refill_ready\": {}", diff --git a/src/core/rdma.rs b/src/core/rdma.rs index 32b3433..5900d57 100644 --- a/src/core/rdma.rs +++ b/src/core/rdma.rs @@ -92,25 +92,35 @@ pub const CRC_LEN: usize = RDMA_CRC_LEN; pub const FAIL_ALLOC: i32 = RDMA_FAIL_ALLOC; #[allow(non_upper_case_globals)] pub const CRC_MISMATCH: i32 = RDMA_CRC_MISMATCH; - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum RdmaStorageEngineType { #[default] - DRAM = 0, - PMEM = 1, - SSD = 2, - INVALID = 3, + #[serde(alias = "DRAM")] + Dram = 0, + #[serde(alias = "PMEM")] + Pmem = 1, + #[serde(alias = "SSD")] + Ssd = 2, + #[serde(alias = "INVALID")] + Invalid = 3, +} + +#[allow(non_upper_case_globals)] +impl RdmaStorageEngineType { + pub const DRAM: Self = Self::Dram; + pub const PMEM: Self = Self::Pmem; + pub const SSD: Self = Self::Ssd; + pub const INVALID: Self = Self::Invalid; } impl RdmaStorageEngineType { pub fn from_code(code: u8) -> Self { match code { - 0 => Self::DRAM, - 1 => Self::PMEM, - 2 => Self::SSD, - _ => Self::INVALID, + 0 => Self::Dram, + 1 => Self::Pmem, + 2 => Self::Ssd, + _ => Self::Invalid, } } @@ -731,7 +741,7 @@ where continue; } let entry = &self.entries[pos]; - let signature_match = if entry.get_storage_engine_type() == RdmaStorageEngineType::SSD { + let signature_match = if entry.get_storage_engine_type() == RdmaStorageEngineType::Ssd { &entry.signature == sig128 } else { &entry.get_signature_96b() == sig96 @@ -802,7 +812,7 @@ where return RdmaHashTableGet { addr: None, len: 0, - storage_type: RdmaStorageEngineType::INVALID, + storage_type: RdmaStorageEngineType::Invalid, }; } let entry = &bucket.entries[pos as usize]; @@ -836,7 +846,7 @@ where status: RDMA_BUCKET_LOCKED, old_addr: None, old_len: 0, - old_type: RdmaStorageEngineType::INVALID, + old_type: RdmaStorageEngineType::Invalid, }; } let sig96 = signature_96(&key, bucket_pos as u64); @@ -845,7 +855,7 @@ where let block_size = kv_size.saturating_add(RDMA_DATA_HEADER + RDMA_CRC_LEN); let mut old_addr = None; let mut old_len = 0; - let mut old_type = RdmaStorageEngineType::INVALID; + let mut old_type = RdmaStorageEngineType::Invalid; let pos = if existing >= 0 { let pos = existing as usize; let old = &bucket.entries[pos]; @@ -867,7 +877,7 @@ where }; let entry = &mut bucket.entries[pos]; - if storage_type == RdmaStorageEngineType::SSD { + if storage_type == RdmaStorageEngineType::Ssd { entry.set_signature_128(sig128); } else { entry.set_signature_96(sig96); @@ -903,7 +913,7 @@ where status: RDMA_BUCKET_LOCKED, addr: None, len: 0, - storage_type: RdmaStorageEngineType::INVALID, + storage_type: RdmaStorageEngineType::Invalid, }; } let sig96 = signature_96(key, bucket_pos as u64); @@ -915,7 +925,7 @@ where status: RDMA_NOT_FOUND, addr: None, len: 0, - storage_type: RdmaStorageEngineType::INVALID, + storage_type: RdmaStorageEngineType::Invalid, }; } let entry = bucket.entries[pos as usize].clone(); @@ -1045,10 +1055,10 @@ pub struct RdmaStorageEngine { impl RdmaStorageEngine { pub fn new(storage_type: RdmaStorageEngineType, capacity: usize) -> Self { let base = match storage_type { - RdmaStorageEngineType::DRAM => 0x0100_0000, - RdmaStorageEngineType::PMEM => 0x0200_0000, - RdmaStorageEngineType::SSD => 0x0300_0000, - RdmaStorageEngineType::INVALID => 0x0400_0000, + RdmaStorageEngineType::Dram => 0x0100_0000, + RdmaStorageEngineType::Pmem => 0x0200_0000, + RdmaStorageEngineType::Ssd => 0x0300_0000, + RdmaStorageEngineType::Invalid => 0x0400_0000, }; Self { storage_type, @@ -1160,7 +1170,7 @@ pub struct RdmaStorageEngineDram { impl RdmaStorageEngineDram { pub fn new(capacity: usize) -> Self { Self { - inner: RdmaStorageEngine::new(RdmaStorageEngineType::DRAM, capacity), + inner: RdmaStorageEngine::new(RdmaStorageEngineType::Dram, capacity), } } @@ -1225,7 +1235,7 @@ pub struct RdmaStorageEnginePMem { impl RdmaStorageEnginePMem { pub fn new(capacity: usize) -> Self { Self { - inner: RdmaStorageEngine::new(RdmaStorageEngineType::PMEM, capacity), + inner: RdmaStorageEngine::new(RdmaStorageEngineType::Pmem, capacity), } } @@ -1300,7 +1310,7 @@ pub struct RdmaStorageEngineSSD { impl RdmaStorageEngineSSD { pub fn new(capacity: usize) -> Self { Self { - inner: RdmaStorageEngine::new(RdmaStorageEngineType::SSD, capacity), + inner: RdmaStorageEngine::new(RdmaStorageEngineType::Ssd, capacity), } } @@ -1361,18 +1371,28 @@ impl RdmaStorageEngineSSD { #[derive(Default)] pub enum RdmaReplacementPolicyType { #[default] - FIFO = 0, - LRU = 1, - OTHER = 2, + #[serde(alias = "FIFO")] + Fifo = 0, + #[serde(alias = "LRU")] + Lru = 1, + #[serde(alias = "OTHER")] + Other = 2, +} + +#[allow(non_upper_case_globals)] +impl RdmaReplacementPolicyType { + pub const FIFO: Self = Self::Fifo; + pub const LRU: Self = Self::Lru; + pub const OTHER: Self = Self::Other; } impl RdmaReplacementPolicyType { pub fn as_replacement_policy_type(self) -> ReplacementPolicyType { match self { - Self::FIFO => ReplacementPolicyType::kFIFO, - Self::LRU => ReplacementPolicyType::kLRU, - Self::OTHER => ReplacementPolicyType::kMaxCode, + Self::Fifo => ReplacementPolicyType::Fifo, + Self::Lru => ReplacementPolicyType::Lru, + Self::Other => ReplacementPolicyType::MaxCode, } } } @@ -1403,7 +1423,7 @@ impl RDMACache { } pub fn with_dram_capacity(dram_capacity: usize) -> Self { - Self::new(dram_capacity, 0, 0, RdmaReplacementPolicyType::FIFO) + Self::new(dram_capacity, 0, 0, RdmaReplacementPolicyType::Fifo) } pub fn lookup(&self, key: &[u8], response: &mut RDMAResponse) -> i32 { @@ -1413,22 +1433,22 @@ impl RDMACache { return RDMA_NOT_FOUND; }; match index.storage_type { - RdmaStorageEngineType::DRAM => { + RdmaStorageEngineType::Dram => { self.dram_engine.as_ref().map_or(RDMA_NOT_FOUND, |engine| { engine.get(key, index.len, response, addr) }) } - RdmaStorageEngineType::PMEM => { + RdmaStorageEngineType::Pmem => { self.pmem_engine.as_ref().map_or(RDMA_NOT_FOUND, |engine| { engine.get(key, index.len, response, addr) }) } - RdmaStorageEngineType::SSD => { + RdmaStorageEngineType::Ssd => { self.ssd_engine.as_ref().map_or(RDMA_NOT_FOUND, |engine| { engine.get(key, index.len, response, addr) }) } - RdmaStorageEngineType::INVALID => RDMA_NOT_FOUND, + RdmaStorageEngineType::Invalid => RDMA_NOT_FOUND, } } @@ -1438,7 +1458,7 @@ impl RDMACache { } pub fn insert(&mut self, key: &[u8], value: &[u8]) -> i32 { - self.insert_to_storage(RdmaStorageEngineType::DRAM, key, value) + self.insert_to_storage(RdmaStorageEngineType::Dram, key, value) } #[allow(non_snake_case)] @@ -1503,19 +1523,19 @@ impl RDMACache { pub fn get_capacity(&self, storage_type: RdmaStorageEngineType) -> usize { match storage_type { - RdmaStorageEngineType::DRAM => self + RdmaStorageEngineType::Dram => self .dram_engine .as_ref() .map_or(0, |engine| engine.stats().0), - RdmaStorageEngineType::PMEM => self + RdmaStorageEngineType::Pmem => self .pmem_engine .as_ref() .map_or(0, |engine| engine.stats().0), - RdmaStorageEngineType::SSD => self + RdmaStorageEngineType::Ssd => self .ssd_engine .as_ref() .map_or(0, |engine| engine.stats().0), - RdmaStorageEngineType::INVALID => 0, + RdmaStorageEngineType::Invalid => 0, } } @@ -1526,16 +1546,16 @@ impl RDMACache { pub fn init_storage_engine(&mut self, storage_type: RdmaStorageEngineType, capacity: usize) { match storage_type { - RdmaStorageEngineType::DRAM => { + RdmaStorageEngineType::Dram => { self.dram_engine = Some(RdmaStorageEngineDram::new(capacity)); } - RdmaStorageEngineType::PMEM => { + RdmaStorageEngineType::Pmem => { self.pmem_engine = Some(RdmaStorageEnginePMem::new(capacity)); } - RdmaStorageEngineType::SSD => { + RdmaStorageEngineType::Ssd => { self.ssd_engine = Some(RdmaStorageEngineSSD::new(capacity)); } - RdmaStorageEngineType::INVALID => {} + RdmaStorageEngineType::Invalid => {} } } @@ -1571,14 +1591,14 @@ impl RDMACache { storage_type: RdmaStorageEngineType, ) -> Option<(usize, usize, usize)> { match storage_type { - RdmaStorageEngineType::DRAM => { + RdmaStorageEngineType::Dram => { self.dram_engine.as_ref().map(RdmaStorageEngineDram::stats) } - RdmaStorageEngineType::PMEM => { + RdmaStorageEngineType::Pmem => { self.pmem_engine.as_ref().map(RdmaStorageEnginePMem::stats) } - RdmaStorageEngineType::SSD => self.ssd_engine.as_ref().map(RdmaStorageEngineSSD::stats), - RdmaStorageEngineType::INVALID => None, + RdmaStorageEngineType::Ssd => self.ssd_engine.as_ref().map(RdmaStorageEngineSSD::stats), + RdmaStorageEngineType::Invalid => None, } } @@ -1589,10 +1609,10 @@ impl RDMACache { value: &[u8], ) -> Option { match storage_type { - RdmaStorageEngineType::DRAM => self.dram_engine.as_mut()?.put(key, value), - RdmaStorageEngineType::PMEM => self.pmem_engine.as_mut()?.put(key, value), - RdmaStorageEngineType::SSD => self.ssd_engine.as_mut()?.put(key, value), - RdmaStorageEngineType::INVALID => None, + RdmaStorageEngineType::Dram => self.dram_engine.as_mut()?.put(key, value), + RdmaStorageEngineType::Pmem => self.pmem_engine.as_mut()?.put(key, value), + RdmaStorageEngineType::Ssd => self.ssd_engine.as_mut()?.put(key, value), + RdmaStorageEngineType::Invalid => None, } } @@ -1603,19 +1623,19 @@ impl RDMACache { len: usize, ) -> i32 { match storage_type { - RdmaStorageEngineType::DRAM => self + RdmaStorageEngineType::Dram => self .dram_engine .as_mut() .map_or(RDMA_NOT_FOUND, |engine| engine.del(addr, len)), - RdmaStorageEngineType::PMEM => self + RdmaStorageEngineType::Pmem => self .pmem_engine .as_mut() .map_or(RDMA_NOT_FOUND, |engine| engine.del(addr, len)), - RdmaStorageEngineType::SSD => self + RdmaStorageEngineType::Ssd => self .ssd_engine .as_mut() .map_or(RDMA_NOT_FOUND, |engine| engine.del(addr, len)), - RdmaStorageEngineType::INVALID => RDMA_NOT_FOUND, + RdmaStorageEngineType::Invalid => RDMA_NOT_FOUND, } } } diff --git a/src/core/storage_config.rs b/src/core/storage_config.rs index 64e469d..eb12ed9 100644 --- a/src/core/storage_config.rs +++ b/src/core/storage_config.rs @@ -12,84 +12,144 @@ pub struct CacheRecoverReport { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CacheTier { Memory, + #[serde(alias = "kPMEM")] Pmem, + #[serde(alias = "kSSD")] Ssd, Reject, } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum CacheInstanceType { - kDRAM = 0, - kPMEM = 1, - kSSD = 2, - kUnified = 3, + #[serde(alias = "kDRAM")] + Dram = 0, + #[serde(alias = "kPMEM")] + Pmem = 1, + #[serde(alias = "kSSD")] + Ssd = 2, + #[serde(alias = "kUnified")] + Unified = 3, +} + +#[allow(non_upper_case_globals)] +impl CacheInstanceType { + pub const kDRAM: Self = Self::Dram; + pub const kPMEM: Self = Self::Pmem; + pub const kSSD: Self = Self::Ssd; + pub const kUnified: Self = Self::Unified; } impl CacheInstanceType { fn as_tier(self) -> Option { match self { - CacheInstanceType::kDRAM => Some(CacheTier::Memory), - CacheInstanceType::kPMEM => Some(CacheTier::Pmem), - CacheInstanceType::kSSD => Some(CacheTier::Ssd), - CacheInstanceType::kUnified => None, + CacheInstanceType::Dram => Some(CacheTier::Memory), + CacheInstanceType::Pmem => Some(CacheTier::Pmem), + CacheInstanceType::Ssd => Some(CacheTier::Ssd), + CacheInstanceType::Unified => None, } } } - -#[allow(non_camel_case_types)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum StorageEngineType { - kDRAM, - kPMEM, - kSSD, - kSimple, - kMultiSSD, + #[serde(alias = "kDRAM")] + Dram, + #[serde(alias = "kPMEM")] + Pmem, + #[serde(alias = "kSSD")] + Ssd, + #[serde(alias = "kSimple")] + Simple, + #[serde(alias = "kMultiSSD")] + MultiSsd, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl StorageEngineType { + pub const kDRAM: Self = Self::Dram; + pub const kPMEM: Self = Self::Pmem; + pub const kSSD: Self = Self::Ssd; + pub const kSimple: Self = Self::Simple; + pub const kMultiSSD: Self = Self::MultiSsd; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum SSDEngineType { - /// Supported Rust SSD engine. - kRocksDB = 0, + /// Supported Rust Ssd engine. + #[serde(alias = "kRocksDB")] + RocksDb = 0, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl SSDEngineType { + pub const kRocksDB: Self = Self::RocksDb; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum WriteBufferType { - kUserDataBuf = 0, - kMetaDataBuf = 1, - kGCBuf = 2, - kCodecDataBuf = 3, + #[serde(alias = "kUserDataBuf")] + UserDataBuf = 0, + #[serde(alias = "kMetaDataBuf")] + MetaDataBuf = 1, + #[serde(alias = "kGCBuf")] + GcBuf = 2, + #[serde(alias = "kCodecDataBuf")] + CodecDataBuf = 3, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl WriteBufferType { + pub const kUserDataBuf: Self = Self::UserDataBuf; + pub const kMetaDataBuf: Self = Self::MetaDataBuf; + pub const kGCBuf: Self = Self::GcBuf; + pub const kCodecDataBuf: Self = Self::CodecDataBuf; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum DataType { - DATA = 1, - META_LOG = 2, + #[serde(alias = "DATA")] + Data = 1, + #[serde(alias = "META_LOG")] + MetaLog = 2, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl DataType { + pub const DATA: Self = Self::Data; + pub const META_LOG: Self = Self::MetaLog; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum GCMode { - LOSSY = 1, - LOSSLESS = 10, + #[serde(alias = "LOSSY")] + Lossy = 1, + #[serde(alias = "LOSSLESS")] + Lossless = 10, } -#[allow(non_camel_case_types)] +#[allow(non_upper_case_globals)] +impl GCMode { + pub const LOSSY: Self = Self::Lossy; + pub const LOSSLESS: Self = Self::Lossless; +} #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum RecordStateType { - kSoftDel = 0x0, - kNormal = 0x1, - kPinned = 0x2, - kMaxCode = 0xf, + #[serde(alias = "kSoftDel")] + SoftDel = 0x0, + #[serde(alias = "kNormal")] + Normal = 0x1, + #[serde(alias = "kPinned")] + Pinned = 0x2, + #[serde(alias = "kMaxCode")] + MaxCode = 0xf, +} + +#[allow(non_upper_case_globals)] +impl RecordStateType { + pub const kSoftDel: Self = Self::SoftDel; + pub const kNormal: Self = Self::Normal; + pub const kPinned: Self = Self::Pinned; + pub const kMaxCode: Self = Self::MaxCode; } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -155,8 +215,8 @@ impl SsdIndex { let mut entries = self.entries.write().expect("ssd index lock poisoned"); let value = entries.get_mut(key)?; if let SsdIndexValue::Memory { state, .. } = value { - if *state == RecordStateType::kSoftDel { - *state = RecordStateType::kNormal; + if *state == RecordStateType::SoftDel { + *state = RecordStateType::Normal; } } Some(value.clone()) @@ -165,8 +225,8 @@ impl SsdIndex { pub fn unpin(&self, key: &str) { let mut entries = self.entries.write().expect("ssd index lock poisoned"); if let Some(SsdIndexValue::Memory { state, .. }) = entries.get_mut(key) { - if *state == RecordStateType::kPinned { - *state = RecordStateType::kNormal; + if *state == RecordStateType::Pinned { + *state = RecordStateType::Normal; } } } @@ -176,18 +236,18 @@ impl SsdIndex { let Some(SsdIndexValue::Memory { state, .. }) = entries.get_mut(key) else { return false; }; - if *state == RecordStateType::kPinned { + if *state == RecordStateType::Pinned { return false; } - *state = RecordStateType::kPinned; + *state = RecordStateType::Pinned; true } pub fn soft_delete(&self, key: &str) { let mut entries = self.entries.write().expect("ssd index lock poisoned"); if let Some(SsdIndexValue::Memory { state, .. }) = entries.get_mut(key) { - if *state != RecordStateType::kPinned { - *state = RecordStateType::kSoftDel; + if *state != RecordStateType::Pinned { + *state = RecordStateType::SoftDel; } } } @@ -417,7 +477,7 @@ impl WriteBuffer { impl Default for WriteBuffer { fn default() -> Self { - Self::new(WriteBufferType::kUserDataBuf, 10_485_760) + Self::new(WriteBufferType::UserDataBuf, 10_485_760) } } @@ -522,7 +582,7 @@ pub fn mask_colored_ptr_size(old_colored_ptr: u64, size: u32) -> u64 { } pub fn mask_colored_ptr_record_state(old_colored_ptr: u64, state: RecordStateType) -> u64 { - old_colored_ptr | ((state as u64) & (RecordStateType::kMaxCode as u64)) + old_colored_ptr | ((state as u64) & (RecordStateType::MaxCode as u64)) } #[allow(non_snake_case)] @@ -718,7 +778,7 @@ impl BufferEncoder { let record_size = Self::DATA_FIXED_PART_SIZE.saturating_add(value_len); let record_units = aligned_to(record_size, self.align_size) / self.align_size as u32; let mut colored_ptr = 0; - colored_ptr = mask_colored_ptr_record_state(colored_ptr, RecordStateType::kSoftDel); + colored_ptr = mask_colored_ptr_record_state(colored_ptr, RecordStateType::SoftDel); colored_ptr = mask_colored_ptr_lba(colored_ptr, batch_begin_offset); colored_ptr = mask_colored_ptr_size(colored_ptr, record_units); update_entry_cb(&record.key, SsdIndexValue::SsdColoredPtr(colored_ptr)); @@ -978,44 +1038,44 @@ impl BufferManager { } impl SSDEngineType { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("rocksdb") || value.eq_ignore_ascii_case("rocks_db") || value.eq_ignore_ascii_case("kRocksDB") || value.eq_ignore_ascii_case("kSSDRocksDBStorageEngine") { - Self::kRocksDB + Self::RocksDb } else { - Self::kRocksDB + Self::RocksDb } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kRocksDB => "RocksDB", + Self::RocksDb => "RocksDB", } } #[allow(non_snake_case)] - pub fn FromReferenceName(value: &str) -> Self { - Self::from_reference_name(value) + pub fn FromConfigName(value: &str) -> Self { + Self::from_config_name(value) } #[allow(non_snake_case)] - pub fn AsReferenceName(self) -> &'static str { - self.as_reference_name() + pub fn AsConfigName(self) -> &'static str { + self.as_config_name() } } impl StorageEngineType { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("pmem") || value.eq_ignore_ascii_case("persistent_memory") || value.eq_ignore_ascii_case("persistent-memory") || value.eq_ignore_ascii_case("kPMEMStorageEngine") || value.eq_ignore_ascii_case("kPMEM") { - Self::kPMEM + Self::Pmem } else if value.eq_ignore_ascii_case("ssd") || value.eq_ignore_ascii_case("rocksdb") || value.eq_ignore_ascii_case("rocks_db") @@ -1023,76 +1083,76 @@ impl StorageEngineType { || value.eq_ignore_ascii_case("kSSDRocksDBStorageEngine") || value.eq_ignore_ascii_case("kSSD") { - Self::kSSD + Self::Ssd } else if value.eq_ignore_ascii_case("simple") || value.eq_ignore_ascii_case("simple_storage") || value.eq_ignore_ascii_case("kSimpleStorageEngine") { - Self::kSimple + Self::Simple } else if value.eq_ignore_ascii_case("multi_ssd") || value.eq_ignore_ascii_case("multi-ssd") || value.eq_ignore_ascii_case("kMultiSSDStorageEngine") { - Self::kMultiSSD + Self::MultiSsd } else if value.eq_ignore_ascii_case("dram") || value.eq_ignore_ascii_case("kDRAM") || value.eq_ignore_ascii_case("kDRAMStorageEngine") { - Self::kDRAM + Self::Dram } else { - Self::kDRAM + Self::Dram } } - pub fn from_reference_code(value: u8) -> Self { + pub fn from_config_code(value: u8) -> Self { match value { - 1 => Self::kPMEM, - 2 => Self::kSSD, - 3 => Self::kSimple, - 4 => Self::kMultiSSD, - _ => Self::kDRAM, + 1 => Self::Pmem, + 2 => Self::Ssd, + 3 => Self::Simple, + 4 => Self::MultiSsd, + _ => Self::Dram, } } - pub fn reference_code(self) -> u8 { + pub fn config_code(self) -> u8 { match self { - Self::kDRAM => 0, - Self::kPMEM => 1, - Self::kSSD => 2, - Self::kSimple => 3, - Self::kMultiSSD => 4, + Self::Dram => 0, + Self::Pmem => 1, + Self::Ssd => 2, + Self::Simple => 3, + Self::MultiSsd => 4, } } pub fn is_ssd_like(self) -> bool { - matches!(self, Self::kSSD | Self::kMultiSSD) + matches!(self, Self::Ssd | Self::MultiSsd) } pub fn canonical_instance_type(self) -> CacheInstanceType { match self { - Self::kDRAM | Self::kSimple => CacheInstanceType::kDRAM, - Self::kPMEM => CacheInstanceType::kPMEM, - Self::kSSD | Self::kMultiSSD => CacheInstanceType::kSSD, + Self::Dram | Self::Simple => CacheInstanceType::Dram, + Self::Pmem => CacheInstanceType::Pmem, + Self::Ssd | Self::MultiSsd => CacheInstanceType::Ssd, } } - pub fn as_reference_enum_name(self) -> &'static str { + pub fn as_config_enum_name(self) -> &'static str { match self { - Self::kDRAM => "kDRAMStorageEngine", - Self::kPMEM => "kPMEMStorageEngine", - Self::kSSD => "kSSDRocksDBStorageEngine", - Self::kSimple => "kSimpleStorageEngine", - Self::kMultiSSD => "kMultiSSDStorageEngine", + Self::Dram => "kDRAMStorageEngine", + Self::Pmem => "kPMEMStorageEngine", + Self::Ssd => "kSSDRocksDBStorageEngine", + Self::Simple => "kSimpleStorageEngine", + Self::MultiSsd => "kMultiSSDStorageEngine", } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kDRAM => "DRAM", - Self::kPMEM => "PMEM", - Self::kSSD => "SSD", - Self::kSimple => "Simple", - Self::kMultiSSD => "MultiSSD", + Self::Dram => "DRAM", + Self::Pmem => "PMEM", + Self::Ssd => "SSD", + Self::Simple => "Simple", + Self::MultiSsd => "MultiSSD", } } @@ -1101,13 +1161,13 @@ impl StorageEngineType { } #[allow(non_snake_case)] - pub fn FromReferenceCode(value: u8) -> Self { - Self::from_reference_code(value) + pub fn FromConfigCode(value: u8) -> Self { + Self::from_config_code(value) } #[allow(non_snake_case)] - pub fn ReferenceCode(self) -> u8 { - self.reference_code() + pub fn ConfigCode(self) -> u8 { + self.config_code() } #[allow(non_snake_case)] @@ -1116,73 +1176,87 @@ impl StorageEngineType { } #[allow(non_snake_case)] - pub fn AsReferenceEnumName(self) -> &'static str { - self.as_reference_enum_name() + pub fn AsConfigEnumName(self) -> &'static str { + self.as_config_enum_name() } #[allow(non_snake_case)] - pub fn AsReferenceName(self) -> &'static str { - self.as_reference_name() + pub fn AsConfigName(self) -> &'static str { + self.as_config_name() } } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum ReplacementPolicyType { - kFIFO = 0, - kLRU = 1, - kSLRU = 2, - kWeightedHotnessLru = 3, - kMaxCode = 4, + #[serde(alias = "kFIFO")] + Fifo = 0, + #[serde(alias = "kLRU")] + Lru = 1, + #[serde(alias = "kSLRU")] + Slru = 2, + #[serde(alias = "kWeightedHotnessLru")] + WeightedHotnessLru = 3, + #[serde(alias = "kMaxCode")] + MaxCode = 4, +} + +#[allow(non_upper_case_globals)] +impl ReplacementPolicyType { + pub const kFIFO: Self = Self::Fifo; + pub const kLRU: Self = Self::Lru; + pub const kSLRU: Self = Self::Slru; + pub const kWeightedHotnessLru: Self = Self::WeightedHotnessLru; + pub const kMaxCode: Self = Self::MaxCode; } impl ReplacementPolicyType { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("fifo") || value.eq_ignore_ascii_case("kFIFO") { - Self::kFIFO + Self::Fifo } else if value.eq_ignore_ascii_case("slru") || value.eq_ignore_ascii_case("kSLRU") { - Self::kSLRU + Self::Slru } else if value.eq_ignore_ascii_case("lru") || value.eq_ignore_ascii_case("kLRU") { - Self::kLRU + Self::Lru } else if value.eq_ignore_ascii_case("kMaxCode") { - Self::kMaxCode + Self::MaxCode } else { - Self::kWeightedHotnessLru + Self::WeightedHotnessLru } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kFIFO => "FIFO", - Self::kSLRU => "SLRU", - Self::kLRU => "LRU", - Self::kWeightedHotnessLru => "WeightedHotnessLru", - Self::kMaxCode => "MaxCode", + Self::Fifo => "FIFO", + Self::Slru => "SLRU", + Self::Lru => "LRU", + Self::WeightedHotnessLru => "WeightedHotnessLru", + Self::MaxCode => "MaxCode", } } fn as_cache_policy(self) -> CacheReplacementPolicy { match self { - ReplacementPolicyType::kFIFO => CacheReplacementPolicy::Fifo, - ReplacementPolicyType::kSLRU => CacheReplacementPolicy::Slru, - ReplacementPolicyType::kLRU => CacheReplacementPolicy::WeightedHotnessLru, - ReplacementPolicyType::kWeightedHotnessLru => { + ReplacementPolicyType::Fifo => CacheReplacementPolicy::Fifo, + ReplacementPolicyType::Slru => CacheReplacementPolicy::Slru, + ReplacementPolicyType::Lru => CacheReplacementPolicy::WeightedHotnessLru, + ReplacementPolicyType::WeightedHotnessLru => { CacheReplacementPolicy::WeightedHotnessLru } - ReplacementPolicyType::kMaxCode => CacheReplacementPolicy::WeightedHotnessLru, + ReplacementPolicyType::MaxCode => CacheReplacementPolicy::WeightedHotnessLru, } } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CacheDataPlacement { + #[serde(alias = "kSideBySide")] SideBySide, + #[serde(alias = "kTiered")] Tiered, } impl CacheDataPlacement { - pub fn try_from_reference_name(value: &str) -> Result { + pub fn try_from_config_name(value: &str) -> Result { if value.eq_ignore_ascii_case("sidebyside") || value.eq_ignore_ascii_case("side_by_side") || value.eq_ignore_ascii_case("side-by-side") @@ -1202,7 +1276,7 @@ impl CacheDataPlacement { } } - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("sidebyside") || value.eq_ignore_ascii_case("side_by_side") || value.eq_ignore_ascii_case("side-by-side") @@ -1213,75 +1287,83 @@ impl CacheDataPlacement { } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { Self::SideBySide => "SideBySide", Self::Tiered => "Tiered", } } } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum DRAMPMEMDataPlacementType { - kSideBySide = 0, - kTiered = 1, - kMaxCode = 2, + #[serde(alias = "kSideBySide")] + SideBySide = 0, + #[serde(alias = "kTiered")] + Tiered = 1, + #[serde(alias = "kMaxCode")] + MaxCode = 2, +} + +#[allow(non_upper_case_globals)] +impl DRAMPMEMDataPlacementType { + pub const kSideBySide: Self = Self::SideBySide; + pub const kTiered: Self = Self::Tiered; + pub const kMaxCode: Self = Self::MaxCode; } impl DRAMPMEMDataPlacementType { - pub fn try_from_reference_name(value: &str) -> Result { - Ok(match CacheDataPlacement::try_from_reference_name(value)? { - CacheDataPlacement::SideBySide => Self::kSideBySide, - CacheDataPlacement::Tiered => Self::kTiered, + pub fn try_from_config_name(value: &str) -> Result { + Ok(match CacheDataPlacement::try_from_config_name(value)? { + CacheDataPlacement::SideBySide => Self::SideBySide, + CacheDataPlacement::Tiered => Self::Tiered, }) } - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("sidebyside") || value.eq_ignore_ascii_case("side_by_side") || value.eq_ignore_ascii_case("side-by-side") || value.eq_ignore_ascii_case("kSideBySide") { - Self::kSideBySide + Self::SideBySide } else if value.eq_ignore_ascii_case("kMaxCode") { - Self::kMaxCode + Self::MaxCode } else { - Self::kTiered + Self::Tiered } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kSideBySide => "SideBySide", - Self::kTiered => "Tiered", - Self::kMaxCode => "MaxCode", + Self::SideBySide => "SideBySide", + Self::Tiered => "Tiered", + Self::MaxCode => "MaxCode", } } pub fn as_cache_data_placement(self) -> CacheDataPlacement { match self { - Self::kSideBySide => CacheDataPlacement::SideBySide, - Self::kTiered | Self::kMaxCode => CacheDataPlacement::Tiered, + Self::SideBySide => CacheDataPlacement::SideBySide, + Self::Tiered | Self::MaxCode => CacheDataPlacement::Tiered, } } pub fn from_cache_data_placement(placement: CacheDataPlacement) -> Self { match placement { - CacheDataPlacement::SideBySide => Self::kSideBySide, - CacheDataPlacement::Tiered => Self::kTiered, + CacheDataPlacement::SideBySide => Self::SideBySide, + CacheDataPlacement::Tiered => Self::Tiered, } } #[allow(non_snake_case)] - pub fn FromReferenceName(value: &str) -> Self { - Self::from_reference_name(value) + pub fn FromConfigName(value: &str) -> Self { + Self::from_config_name(value) } #[allow(non_snake_case)] - pub fn AsReferenceName(self) -> &'static str { - self.as_reference_name() + pub fn AsConfigName(self) -> &'static str { + self.as_config_name() } #[allow(non_snake_case)] @@ -1320,8 +1402,11 @@ pub enum CacheAdmissionReason { #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CacheAccessRecordType { + #[serde(alias = "kPut")] Put = 1, + #[serde(alias = "kGet")] Get = 2, + #[serde(alias = "kDelete")] Delete = 3, } @@ -1332,11 +1417,11 @@ impl CacheAccessRecordType { pub const kDelete: Self = Self::Delete; pub const kMaxCode: u8 = 4; - pub fn reference_code(self) -> u8 { + pub fn config_code(self) -> u8 { self as u8 } - pub fn from_reference_code(code: u8) -> Option { + pub fn from_config_code(code: u8) -> Option { match code { 1 => Some(Self::Put), 2 => Some(Self::Get), @@ -1345,7 +1430,7 @@ impl CacheAccessRecordType { } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { Self::Put => "kPut", Self::Get => "kGet", @@ -1354,18 +1439,18 @@ impl CacheAccessRecordType { } #[allow(non_snake_case)] - pub fn ReferenceCode(self) -> u8 { - self.reference_code() + pub fn ConfigCode(self) -> u8 { + self.config_code() } #[allow(non_snake_case)] - pub fn FromReferenceCode(code: u8) -> Option { - Self::from_reference_code(code) + pub fn FromConfigCode(code: u8) -> Option { + Self::from_config_code(code) } #[allow(non_snake_case)] - pub fn AsReferenceName(self) -> &'static str { - self.as_reference_name() + pub fn AsConfigName(self) -> &'static str { + self.as_config_name() } } diff --git a/src/lib.rs b/src/lib.rs index fd724bf..408ad11 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,7 @@ // Copyright 2026 MatrixArkAI //! MatrixCache is a Rust-native multi-tier cache library. It manages a hot -//! in-memory (DRAM) tier, a persistent-memory-like resident tier, and an SSD tier +//! in-memory (Dram) tier, a persistent-memory-like resident tier, and an Ssd tier //! (RocksDB by default), with admission control, cross-tier eviction, read-through //! refill, pinned handles, invalidation, and asynchronous writeback with //! backpressure accounting. @@ -12,7 +12,7 @@ //! ```no_run //! use matrixcache::{CacheKey, MultiLayerCache}; //! -//! // 1 MiB in-memory tier; the SSD tier is persisted under `dir`. +//! // 1 MiB in-memory tier; the Ssd tier is persisted under `dir`. //! let dir = std::env::temp_dir().join("matrixcache-doc"); //! let cache = MultiLayerCache::new(1 << 20, dir); //! @@ -22,7 +22,7 @@ //! # Ok::<(), matrixcache::CacheError>(()) //! ``` //! -//! The SSD backend is RocksDB via the default `rocksdb-ssd` feature; build with +//! The Ssd backend is RocksDB via the default `rocksdb-ssd` feature; build with //! `--no-default-features` for a lightweight file-backed compatibility store. use std::collections::hash_map::DefaultHasher; diff --git a/src/runtime/allocators_executors.rs b/src/runtime/allocators_executors.rs index 3f176e1..e03dec0 100644 --- a/src/runtime/allocators_executors.rs +++ b/src/runtime/allocators_executors.rs @@ -33,47 +33,57 @@ impl AllocatorStats { self.num_occupied_bytes } } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum AllocatorType { - kLogBasedAllocator = 0, - kPoolBasedAllocator = 1, - kJeAllocator = 2, - kMaxCode = 3, + #[serde(alias = "kLogBasedAllocator")] + LogBasedAllocator = 0, + #[serde(alias = "kPoolBasedAllocator")] + PoolBasedAllocator = 1, + #[serde(alias = "kJeAllocator")] + JeAllocator = 2, + #[serde(alias = "kMaxCode")] + MaxCode = 3, +} + +#[allow(non_upper_case_globals)] +impl AllocatorType { + pub const kLogBasedAllocator: Self = Self::LogBasedAllocator; + pub const kPoolBasedAllocator: Self = Self::PoolBasedAllocator; + pub const kJeAllocator: Self = Self::JeAllocator; + pub const kMaxCode: Self = Self::MaxCode; } impl AllocatorType { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("log") || value.eq_ignore_ascii_case("log_based") || value.eq_ignore_ascii_case("logbased") || value.eq_ignore_ascii_case("kLogBasedAllocator") { - Self::kLogBasedAllocator + Self::LogBasedAllocator } else if value.eq_ignore_ascii_case("pool") || value.eq_ignore_ascii_case("pool_based") || value.eq_ignore_ascii_case("poolbased") || value.eq_ignore_ascii_case("kPoolBasedAllocator") { - Self::kPoolBasedAllocator + Self::PoolBasedAllocator } else if value.eq_ignore_ascii_case("je") || value.eq_ignore_ascii_case("jemalloc") || value.eq_ignore_ascii_case("kJeAllocator") { - Self::kJeAllocator + Self::JeAllocator } else { - Self::kMaxCode + Self::MaxCode } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kLogBasedAllocator => "LogBasedAllocator", - Self::kPoolBasedAllocator => "PoolBasedAllocator", - Self::kJeAllocator => "JeAllocator", - Self::kMaxCode => "MaxCode", + Self::LogBasedAllocator => "LogBasedAllocator", + Self::PoolBasedAllocator => "PoolBasedAllocator", + Self::JeAllocator => "JeAllocator", + Self::MaxCode => "MaxCode", } } } @@ -91,38 +101,46 @@ pub struct PoolChunkMeta { pub id: ChunkID, pub num_alloc_objects: usize, } - -#[allow(non_camel_case_types)] #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum FlushPolicy { - kNoFlush = 0, - kInstantFlush = 1, - kMiniBatchFlush = 2, + #[serde(alias = "kNoFlush")] + NoFlush = 0, + #[serde(alias = "kInstantFlush")] + InstantFlush = 1, + #[serde(alias = "kMiniBatchFlush")] + MiniBatchFlush = 2, +} + +#[allow(non_upper_case_globals)] +impl FlushPolicy { + pub const kNoFlush: Self = Self::NoFlush; + pub const kInstantFlush: Self = Self::InstantFlush; + pub const kMiniBatchFlush: Self = Self::MiniBatchFlush; } impl FlushPolicy { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("no_flush") || value.eq_ignore_ascii_case("noflush") || value.eq_ignore_ascii_case("kNoFlush") { - Self::kNoFlush + Self::NoFlush } else if value.eq_ignore_ascii_case("instant_flush") || value.eq_ignore_ascii_case("instant") || value.eq_ignore_ascii_case("kInstantFlush") { - Self::kInstantFlush + Self::InstantFlush } else { - Self::kMiniBatchFlush + Self::MiniBatchFlush } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { - Self::kNoFlush => "NoFlush", - Self::kInstantFlush => "InstantFlush", - Self::kMiniBatchFlush => "MiniBatchFlush", + Self::NoFlush => "NoFlush", + Self::InstantFlush => "InstantFlush", + Self::MiniBatchFlush => "MiniBatchFlush", } } } @@ -213,7 +231,7 @@ fn free_virtual_region(ptr: AllocatorPtr) -> Result, CacheError> { } pub fn parse_allocator_type(allocator_type: &str) -> AllocatorType { - AllocatorType::from_reference_name(allocator_type) + AllocatorType::from_config_name(allocator_type) } pub fn dram_allocate_object( diff --git a/src/runtime/builder_and_gc.rs b/src/runtime/builder_and_gc.rs index 409f8b2..f567cb8 100644 --- a/src/runtime/builder_and_gc.rs +++ b/src/runtime/builder_and_gc.rs @@ -689,7 +689,7 @@ impl CacheInner { } } self.pmem.clear(); - self.pmem_fifo_order.clear(); + self.pmem_order.clear(); self.pmem_bytes = 0; for (key, expected_len) in live { report.scanned_files = report.scanned_files.saturating_add(1); @@ -761,7 +761,7 @@ impl CacheInner { }); } self.disk_index = recovered_index; - self.disk_fifo_order = recovered_order.into_iter().collect(); + self.disk_order = recovered_order.into_iter().collect(); self.ssd_bytes = recovered_bytes; self.stats.disk_bytes = recovered_bytes; Ok(report) @@ -836,7 +836,7 @@ impl CacheInner { } self.disk_index = recovered_index; - self.disk_fifo_order = recovered_order.into_iter().collect(); + self.disk_order = recovered_order.into_iter().collect(); self.ssd_bytes = recovered_bytes; self.stats.disk_bytes = recovered_bytes; Ok(report) @@ -904,28 +904,52 @@ impl CacheInner { fn record_hit_metadata(&mut self, key: &CacheKey, block_bytes: usize) { self.access_epoch = self.access_epoch.saturating_add(1); - let block_kind = infer_block_kind(key); - let entry = self.metadata.entry(key.clone()).or_insert(CacheEntryMeta { - block_kind, - routing_slot: extract_routing_slot(key), - hotness: initial_hotness(block_kind, block_bytes), - hits: 0, - last_access_epoch: 0, - admission_reason: CacheAdmissionReason::MemoryOnly, - }); - entry.hits = entry.hits.saturating_add(1); - let before = entry.hotness; - entry.hotness = entry.hotness.saturating_add(1); - entry.last_access_epoch = self.access_epoch; - if before < self.tiering_policy.memory_hotness_threshold - && entry.hotness >= self.tiering_policy.memory_hotness_threshold - { + let epoch = self.access_epoch; + let threshold = self.tiering_policy.memory_hotness_threshold; + + // The hit path runs for every read, so it must not pay for the miss + // path. Looking the entry up first keeps the key clone, the block-kind + // inference and the routing-slot extraction on the branch that + // actually needs them; going through `entry()` did all of that on + // every hit and then discarded it. + let crossed_threshold = if let Some(entry) = self.metadata.get_mut(key) { + entry.hits = entry.hits.saturating_add(1); + let before = entry.hotness; + entry.hotness = entry.hotness.saturating_add(1); + entry.last_access_epoch = epoch; + before < threshold && entry.hotness >= threshold + } else { + let block_kind = infer_block_kind(key); + let before = initial_hotness(block_kind, block_bytes); + let hotness = before.saturating_add(1); + self.metadata.insert( + key.clone(), + CacheEntryMeta { + block_kind, + routing_slot: extract_routing_slot(key), + hotness, + hits: 1, + last_access_epoch: epoch, + admission_reason: CacheAdmissionReason::MemoryOnly, + }, + ); + before < threshold && hotness >= threshold + }; + + if crossed_threshold { self.stats.hotness_promotions = self.stats.hotness_promotions.saturating_add(1); } } fn record_hit(&mut self, key: &CacheKey, block_bytes: usize) { self.record_hit_metadata(key, block_bytes); + // Move the entry to the back of each tier's access order. Victim + // selection reads the front of that order, so without this an entry + // written early stays at the front however often it is read, and is + // offered up for eviction on every pass. + self.memory_order.touch_access(key); + self.pmem_order.touch_access(key); + self.disk_order.touch_access(key); } fn put_memory(&mut self, key: CacheKey, value: Vec) -> bool { @@ -941,7 +965,7 @@ impl CacheInner { if let Some(old) = self.memory.insert(key.clone(), Arc::clone(&value)) { self.memory_bytes = self.memory_bytes.saturating_sub(old.len()); } else { - self.memory_fifo_order.push_back_if_absent(key); + self.memory_order.push_back_if_absent(key); } self.memory_bytes += value.len(); self.evict_memory_to_capacity_since(eviction_started); @@ -977,7 +1001,7 @@ impl CacheInner { if let Some(old) = self.pmem.insert(key.clone(), Arc::clone(&value)) { self.pmem_bytes = self.pmem_bytes.saturating_sub(old.len()); } else { - self.pmem_fifo_order.push_back_if_absent(key); + self.pmem_order.push_back_if_absent(key); } self.pmem_bytes = self.pmem_bytes.saturating_add(value.len()); self.evict_pmem_to_capacity_since(eviction_started); @@ -1021,7 +1045,7 @@ impl CacheInner { return false; } self.disk_index.insert(key.clone(), block_len as u64); - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len as u64); self.stats.disk_fills = self.stats.disk_fills.saturating_add(1); self.stats.ssd_admission_accepted = self.stats.ssd_admission_accepted.saturating_add(1); @@ -1079,7 +1103,6 @@ impl CacheInner { } fn evict_memory_to_capacity_since(&mut self, eviction_started: Instant) { - let mut victim_keys = Vec::new(); while self.memory_bytes > self.memory_capacity_bytes { let before = self.memory_bytes; let Some((victim, reason, pinned_skips)) = self.select_memory_eviction_victim() else { @@ -1091,11 +1114,14 @@ impl CacheInner { .stats .eviction_pinned_skips .saturating_add(pinned_skips); + // Drop the victim from the order as it is taken. Selection reads + // that order, so leaving it until after the loop lets the next + // round pick the same key, whose bytes are already gone. + self.memory_order.remove(&victim); let Some(old_value) = self.memory.remove(&victim) else { - victim_keys.push(victim); - if self.memory_bytes == before { - break; - } + // A key the order still listed but the tier no longer holds. + // It is gone from the order now, so the next round makes + // progress rather than picking it again. continue; }; self.memory_bytes = self.memory_bytes.saturating_sub(old_value.len()); @@ -1111,23 +1137,16 @@ impl CacheInner { { self.metadata.remove(&victim); } - victim_keys.push(victim); self.record_eviction_latency(eviction_started); if self.memory_bytes == before { break; } } - if !victim_keys.is_empty() { - let victim_key_set = victim_keys.iter().cloned().collect::>(); - self.memory_fifo_order - .retain(|candidate| !victim_key_set.contains(candidate)); - } self.stats.memory_bytes = self.memory_bytes as u64; self.refresh_pin_stats(); } fn evict_pmem_to_capacity_since(&mut self, eviction_started: Instant) { - let mut victim_keys = Vec::new(); while self.pmem_bytes > self.pmem_capacity_bytes { let before = self.pmem_bytes; let Some((victim, _reason, pinned_skips)) = self.select_pmem_eviction_victim() else { @@ -1139,11 +1158,14 @@ impl CacheInner { .stats .pmem_eviction_pinned_skips .saturating_add(pinned_skips); + // Drop the victim from the order as it is taken. Selection reads + // that order, so leaving it until after the loop lets the next + // round pick the same key, whose bytes are already gone. + self.pmem_order.remove(&victim); let Some(old_value) = self.pmem.remove(&victim) else { - victim_keys.push(victim); - if self.pmem_bytes == before { - break; - } + // A key the order still listed but the tier no longer holds. + // It is gone from the order now, so the next round makes + // progress rather than picking it again. continue; }; self.pmem_bytes = self.pmem_bytes.saturating_sub(old_value.len()); @@ -1158,17 +1180,11 @@ impl CacheInner { { self.metadata.remove(&victim); } - victim_keys.push(victim); self.record_eviction_latency(eviction_started); if self.pmem_bytes == before { break; } } - if !victim_keys.is_empty() { - let victim_key_set = victim_keys.iter().cloned().collect::>(); - self.pmem_fifo_order - .retain(|candidate| !victim_key_set.contains(candidate)); - } self.stats.pmem_bytes = self.pmem_bytes as u64; self.refresh_pin_stats(); } @@ -1197,7 +1213,7 @@ impl CacheInner { // Drop the victim from the order as it is taken. Selection reads // that order, so leaving it until after the loop lets the next // round pick the same key, whose bytes are already gone. - self.disk_fifo_order.remove(&victim); + self.disk_order.remove(&victim); self.record_eviction(CacheTier::Ssd, victim.clone(), evicted_value.clone()); self.ssd_bytes = self.ssd_bytes.saturating_sub(removed_bytes); self.stats.ssd_evictions = self.stats.ssd_evictions.saturating_add(1); @@ -1218,9 +1234,6 @@ impl CacheInner { self.stats.disk_bytes = self.ssd_bytes; return; } - let victim_key_set = victim_keys.iter().cloned().collect::>(); - self.disk_fifo_order - .retain(|candidate| !victim_key_set.contains(candidate)); let _ = self.delete_ssd_blocks(&victim_keys); for key in &victim_keys { let _ = self.append_disk_manifest_delete(key); @@ -1231,11 +1244,12 @@ impl CacheInner { fn select_memory_eviction_victim(&mut self) -> Option<(CacheKey, EvictionReason, u64)> { match self.memory_replacement_policy { CacheReplacementPolicy::Fifo => { - self.select_fifo_eviction_victim(&self.memory_fifo_order) + self.select_fifo_eviction_victim(&self.memory_order) } CacheReplacementPolicy::Slru | CacheReplacementPolicy::WeightedHotnessLru => { - let keys = self.memory.keys().cloned().collect::>(); - self.select_eviction_victim(keys) + let picked = self.select_windowed_eviction_victim(&self.memory_order); + self.record_sampled_groups(picked.groups_weighed); + picked.victim } } } @@ -1243,11 +1257,12 @@ impl CacheInner { fn select_pmem_eviction_victim(&mut self) -> Option<(CacheKey, EvictionReason, u64)> { match self.pmem_replacement_policy { CacheReplacementPolicy::Fifo => { - self.select_fifo_eviction_victim(&self.pmem_fifo_order) + self.select_fifo_eviction_victim(&self.pmem_order) } CacheReplacementPolicy::Slru | CacheReplacementPolicy::WeightedHotnessLru => { - let keys = self.pmem.keys().cloned().collect::>(); - self.select_eviction_victim(keys) + let picked = self.select_windowed_eviction_victim(&self.pmem_order); + self.record_sampled_groups(picked.groups_weighed); + picked.victim } } } @@ -1255,11 +1270,12 @@ impl CacheInner { fn select_ssd_eviction_victim(&mut self) -> Option<(CacheKey, EvictionReason, u64)> { match self.ssd_replacement_policy { CacheReplacementPolicy::Fifo => { - self.select_fifo_eviction_victim(&self.disk_fifo_order) + self.select_fifo_eviction_victim(&self.disk_order) } CacheReplacementPolicy::Slru | CacheReplacementPolicy::WeightedHotnessLru => { - let keys = self.disk_index.keys().cloned().collect::>(); - self.select_eviction_victim(keys) + let picked = self.select_windowed_eviction_victim(&self.disk_order); + self.record_sampled_groups(picked.groups_weighed); + picked.victim } } } @@ -1279,29 +1295,60 @@ impl CacheInner { None } - fn select_eviction_victim(&mut self, keys: I) -> Option<(CacheKey, EvictionReason, u64)> + /// Weigh a bounded window of `order`, least recently accessed first. + /// + /// Falls back to the whole tier when the window turns up nothing + /// evictable, so a run of pinned entries at the front of the order cannot + /// stall eviction while unpinned entries sit further back. A tier holding + /// no more than the window weighs everything either way, so its victim is + /// exactly the one it would have picked before the window existed. + fn select_windowed_eviction_victim(&self, order: &CacheKeyOrder) -> PickedEvictionVictim { + let windowed = + self.select_eviction_victim(order.iter_access().take(EVICTION_CANDIDATE_WINDOW)); + if windowed.victim.is_some() || order.len() <= EVICTION_CANDIDATE_WINDOW { + return windowed; + } + let full = self.select_eviction_victim(order.iter_access()); + PickedEvictionVictim { + victim: full.victim, + groups_weighed: windowed + .groups_weighed + .saturating_add(full.groups_weighed), + } + } + + fn record_sampled_groups(&mut self, groups: usize) { + self.stats.eviction_sampled_groups = self + .stats + .eviction_sampled_groups + .saturating_add(groups as u64); + } + + /// Weigh `keys` and return the coldest group's coldest member. + /// + /// Borrows the keys rather than taking them by value: the caller passes the + /// tier's own map keys straight in, so a selection no longer starts by + /// cloning every key in the tier. + fn select_eviction_victim<'a, I>(&self, keys: I) -> PickedEvictionVictim where - I: IntoIterator, + I: IntoIterator, { let mut pinned_skips = 0u64; - let mut groups: HashMap = HashMap::new(); + let mut groups: HashMap, SlotEvictionGroup> = HashMap::new(); for key in keys { - if self.pinned.contains_key(&key) { + if self.pinned.contains_key(key) { pinned_skips = pinned_skips.saturating_add(1); continue; } - let score = self.eviction_score(&key); - let group_key = self.eviction_group_key(&key); + let score = self.eviction_score(key); + let group_key = self.eviction_group_key(key); groups .entry(group_key) - .and_modify(|group| group.observe(key.clone(), score)) + .and_modify(|group| group.observe(key, score)) .or_insert_with(|| SlotEvictionGroup::new(key, score)); } - self.stats.eviction_sampled_groups = self - .stats - .eviction_sampled_groups - .saturating_add(groups.len() as u64); - groups + let group_count = groups.len(); + let victim = groups .into_values() .min_by(|left, right| { left.group_score @@ -1315,7 +1362,11 @@ impl CacheInner { eviction_reason_for(group.victim_score), pinned_skips, ) - }) + }); + PickedEvictionVictim { + victim, + groups_weighed: group_count, + } } fn eviction_score(&self, key: &CacheKey) -> EvictionScore { @@ -1348,10 +1399,11 @@ impl CacheInner { let incoming_group = request .routing_slot .or_else(|| extract_routing_slot(key)) - .map(|slot| format!("slot:{slot}")) - .unwrap_or_else(|| format!("object:{}:{}", key.namespace, key.record_key)); - self.disk_index - .keys() + .map(EvictionGroupKey::Slot) + .unwrap_or(EvictionGroupKey::Object(&key.namespace, &key.record_key)); + self.disk_order + .iter() + .take(EVICTION_CANDIDATE_WINDOW) .filter(|candidate| self.eviction_group_key(candidate) != incoming_group) .map(|candidate| self.eviction_score(candidate)) .min() @@ -1359,13 +1411,13 @@ impl CacheInner { .unwrap_or(false) } - fn eviction_group_key(&self, key: &CacheKey) -> String { + fn eviction_group_key<'a>(&self, key: &'a CacheKey) -> EvictionGroupKey<'a> { self.metadata .get(key) .and_then(|meta| meta.routing_slot) .or_else(|| extract_routing_slot(key)) - .map(|slot| format!("slot:{slot}")) - .unwrap_or_else(|| format!("object:{}:{}", key.namespace, key.record_key)) + .map(EvictionGroupKey::Slot) + .unwrap_or(EvictionGroupKey::Object(&key.namespace, &key.record_key)) } fn record_memory_eviction_reason(&mut self, reason: EvictionReason) { @@ -1452,7 +1504,7 @@ impl CacheInner { removed_pinned_bytes = removed_pinned_bytes.max(value.len()); self.memory_bytes = self.memory_bytes.saturating_sub(value.len()); } - self.memory_fifo_order.remove(key); + self.memory_order.remove(key); if remove_disk { let disk_bytes = self.disk_index.remove(key).unwrap_or_default(); removed_pinned_bytes = removed_pinned_bytes.max( @@ -1479,7 +1531,7 @@ impl CacheInner { if remove_disk { let _ = self.persist_pmem_delete(key); } - self.pmem_fifo_order.remove(key); + self.pmem_order.remove(key); self.metadata.remove(key); if key_pinned && removed_pinned_bytes > 0 { self.pinned_removed_bytes @@ -1493,11 +1545,11 @@ impl CacheInner { if remove_disk { match key_set.as_ref() { Some(key_set) => { - self.disk_fifo_order + self.disk_order .retain(|candidate| !key_set.contains(candidate)); } None => { - self.disk_fifo_order + self.disk_order .retain(|candidate| !keys.contains(candidate)); } } @@ -1515,9 +1567,9 @@ impl CacheInner { self.pmem.clear(); self.clear_pmem_persistence()?; self.disk_index.clear(); - self.disk_fifo_order.clear(); - self.memory_fifo_order.clear(); - self.pmem_fifo_order.clear(); + self.disk_order.clear(); + self.memory_order.clear(); + self.pmem_order.clear(); self.pinned.clear(); self.pinned_handle_bytes.clear(); self.pinned_removed_bytes.clear(); @@ -1625,7 +1677,11 @@ impl CacheInner { } fn record_get_latency(&mut self, started: Instant) { - let micros = started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + let micros = elapsed_micros(started); + self.record_get_latency_micros(micros); + } + + fn record_get_latency_micros(&mut self, micros: u64) { self.stats.get_latency_samples = self.stats.get_latency_samples.saturating_add(1); self.stats.get_latency_total_micros = self.stats.get_latency_total_micros.saturating_add(micros); @@ -1661,7 +1717,11 @@ impl CacheInner { } fn record_read_through_latency(&mut self, started: Instant) { - let micros = started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + let micros = elapsed_micros(started); + self.record_read_through_latency_micros(micros); + } + + fn record_read_through_latency_micros(&mut self, micros: u64) { observe_latency_bucket( micros, &mut self.stats.read_through_latency_samples, @@ -1781,6 +1841,16 @@ fn infer_block_kind(key: &CacheKey) -> CacheBlockKind { } } +/// How many candidates victim selection weighs before it settles. +/// +/// Selection used to weigh every resident entry, so a cache sitting at +/// capacity paid a cost proportional to how much it held on every single +/// write. Weighing a bounded window of the oldest-resident entries instead +/// keeps that cost flat as the cache grows. The window is wider than the +/// working set of a small cache, so those keep weighing everything and choose +/// exactly what they chose before. +const EVICTION_CANDIDATE_WINDOW: usize = 512; + fn eviction_reason_for(score: EvictionScore) -> EvictionReason { if score.hotness == 0 { EvictionReason::Cold @@ -1873,3 +1943,8 @@ fn unique_temp_path(kind: &str) -> PathBuf { std::process::id() )) } + +/// Microseconds since , saturating rather than wrapping. +fn elapsed_micros(started: Instant) -> u64 { + started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64 +} diff --git a/src/runtime/cache_facades.rs b/src/runtime/cache_facades.rs index 2746653..12c7c87 100644 --- a/src/runtime/cache_facades.rs +++ b/src/runtime/cache_facades.rs @@ -197,15 +197,15 @@ impl CacheInstance { .cloned() .unwrap_or_else(|| unique_temp_path("cache-instance")); let mut options = match storage_type { - StorageEngineType::kDRAM | StorageEngineType::kSimple => { + StorageEngineType::Dram | StorageEngineType::Simple => { CacheOptions::new(capacity, 0, 0) } - StorageEngineType::kPMEM => CacheOptions::new(0, capacity, 0), - StorageEngineType::kSSD | StorageEngineType::kMultiSSD => { + StorageEngineType::Pmem => CacheOptions::new(0, capacity, 0), + StorageEngineType::Ssd | StorageEngineType::MultiSsd => { CacheOptions::new(0, 0, capacity).with_ssd_instance_only(true) } }; - let ssd_paths = if matches!(storage_type, StorageEngineType::kSSD | StorageEngineType::kMultiSSD) + let ssd_paths = if matches!(storage_type, StorageEngineType::Ssd | StorageEngineType::MultiSsd) && !paths.is_empty() { paths.clone() @@ -216,7 +216,7 @@ impl CacheInstance { instance_type.as_tier().expect("storage-backed instance"), replacement_type.as_cache_policy(), ); - if matches!(storage_type, StorageEngineType::kPMEM) { + if matches!(storage_type, StorageEngineType::Pmem) { options = options.with_pmem_paths(paths); } @@ -478,8 +478,8 @@ impl CacheInstance { pub fn recover_data(&self) -> Result { match self.storage_type { - StorageEngineType::kPMEM => self.cache.recover_pmem_index(), - StorageEngineType::kSSD | StorageEngineType::kMultiSSD => self.cache.recover_disk_index(), + StorageEngineType::Pmem => self.cache.recover_pmem_index(), + StorageEngineType::Ssd | StorageEngineType::MultiSsd => self.cache.recover_disk_index(), _ => Ok(CacheRecoverReport::default()), } } @@ -533,12 +533,12 @@ impl CacheInstance { pub fn get_allocator_type(&self) -> AllocatorType { match self.storage_type { - StorageEngineType::kDRAM | StorageEngineType::kSimple => { - AllocatorType::kPoolBasedAllocator + StorageEngineType::Dram | StorageEngineType::Simple => { + AllocatorType::PoolBasedAllocator } - StorageEngineType::kPMEM => AllocatorType::kLogBasedAllocator, - StorageEngineType::kSSD | StorageEngineType::kMultiSSD => { - AllocatorType::kLogBasedAllocator + StorageEngineType::Pmem => AllocatorType::LogBasedAllocator, + StorageEngineType::Ssd | StorageEngineType::MultiSsd => { + AllocatorType::LogBasedAllocator } } } @@ -1631,7 +1631,10 @@ impl SimpleLRUCache { #[allow(non_snake_case)] pub fn Size(&self) -> usize { - self.inner.lock().expect("simple lru lock poisoned").size + self.inner + .lock() + .expect("simple lru lock poisoned") + .current_size() } } @@ -1756,7 +1759,10 @@ impl ZeroCopySimpleLRUCache { #[allow(non_snake_case)] pub fn Size(&self) -> usize { - self.inner.lock().expect("simple lru lock poisoned").size + self.inner + .lock() + .expect("simple lru lock poisoned") + .current_size() } #[allow(non_snake_case)] @@ -1893,6 +1899,13 @@ struct SimpleLruInner { size: usize, order: CacheKeyOrder, entries: HashMap, + /// Entries removed while a handle still pinned them. + /// + /// Their bytes are still resident, so they keep counting towards the + /// cache size until the last pin is released. Dropping them from the + /// accounting at removal would let the cache admit data it has no room + /// for. + pinned_removed: Vec, } impl SimpleLruInner { @@ -1906,6 +1919,7 @@ impl SimpleLruInner { size: 0, order: CacheKeyOrder::new(), entries: HashMap::new(), + pinned_removed: Vec::new(), } } @@ -1914,6 +1928,7 @@ impl SimpleLruInner { if size > self.capacity { return false; } + self.reap_pinned_removed(); self.remove(&key); self.size += size; self.order.push_front(key.clone()); @@ -1930,15 +1945,46 @@ impl SimpleLruInner { fn remove(&mut self, key: &CacheKey) { if let Some(entry) = self.entries.remove(key) { - self.size = self.size.saturating_sub(entry.size); self.order.remove(key); + self.retire(entry); } } fn remove_all(&mut self) { - self.entries.clear(); + let retired = self.entries.drain().map(|(_, entry)| entry).collect::>(); + for entry in retired { + self.retire(entry); + } self.order.clear(); - self.size = 0; + } + + /// Drop an entry from the index, keeping its bytes counted while a handle + /// still holds the value. + fn retire(&mut self, entry: SimpleLruEntry) { + if Arc::strong_count(&entry.value) > 1 { + self.pinned_removed.push(entry); + } else { + self.size = self.size.saturating_sub(entry.size); + } + } + + /// Release the bytes of any removed entry whose last pin has now dropped. + fn reap_pinned_removed(&mut self) { + let mut released = 0usize; + self.pinned_removed.retain(|entry| { + if Arc::strong_count(&entry.value) > 1 { + return true; + } + released = released.saturating_add(entry.size); + false + }); + self.size = self.size.saturating_sub(released); + } + + /// Current size, after accounting for pins released since the last call. + fn current_size(&mut self) -> usize { + self.reap_pinned_removed(); + self.size } fn set_capacity(&mut self, capacity: usize) { @@ -1957,6 +2003,7 @@ impl SimpleLruInner { } fn evict_unpinned(&mut self) { + self.reap_pinned_removed(); while self.size > self.capacity { // Walk from the least recently used end and stop at the first // entry nobody is holding. Scanning forward for the last match @@ -2342,14 +2389,14 @@ impl FlexibleCache { pmem_paths: impl IntoIterator, ssd_paths: impl IntoIterator, ) -> Self { - let policy = ReplacementPolicyType::from_reference_name(policy.as_ref()); - let engine = StorageEngineType::from_reference_name(engine.as_ref()); + let policy = ReplacementPolicyType::from_config_name(policy.as_ref()); + let engine = StorageEngineType::from_config_name(engine.as_ref()); let paths = match engine { - StorageEngineType::kPMEM => pmem_paths.into_iter().collect::>(), - StorageEngineType::kSSD | StorageEngineType::kMultiSSD => { + StorageEngineType::Pmem => pmem_paths.into_iter().collect::>(), + StorageEngineType::Ssd | StorageEngineType::MultiSsd => { ssd_paths.into_iter().collect::>() } - StorageEngineType::kDRAM | StorageEngineType::kSimple => Vec::new(), + StorageEngineType::Dram | StorageEngineType::Simple => Vec::new(), }; Self { instance: CacheInstance::new(capacity, policy, engine, paths.clone()), @@ -2677,11 +2724,11 @@ impl MultiTierCache { side_by_side_dram_pmem_placement_threshold: usize, ssd_storage_engine: impl AsRef, ) -> Result { - let policy_type = ReplacementPolicyType::from_reference_name(policy.as_ref()); + let policy_type = ReplacementPolicyType::from_config_name(policy.as_ref()); let replacement_policy = policy_type.as_cache_policy(); - let ssd_storage_engine = StorageEngineType::from_reference_name(ssd_storage_engine.as_ref()); + let ssd_storage_engine = StorageEngineType::from_config_name(ssd_storage_engine.as_ref()); let data_placement = - CacheDataPlacement::try_from_reference_name(dram_pmem_data_placement_type.as_ref())?; + CacheDataPlacement::try_from_config_name(dram_pmem_data_placement_type.as_ref())?; let options = CacheOptions::new(dram_capacity, pmem_capacity, ssd_capacity) .with_pmem_paths(pmem_paths) .with_ssd_paths(ssd_paths) @@ -2826,9 +2873,9 @@ impl MultiTierCache { "matrixcache_stats metrics={} comments={} policy={} ssd_engine={} placement={} eviction_enabled={} memory_bytes={} pmem_bytes={} disk_bytes={} pinned_entries={} pinned_bytes={} memory_evictions={} pmem_evictions={} ssd_evictions={} ssd_admission_rejections={} async_writeback_queue_depth={} async_writeback_queue_bytes={} writeback_backpressure_events={}", metrics.as_ref(), comments.as_ref(), - self.policy.as_reference_name(), - self.ssd_storage_engine.as_reference_name(), - self.cache.production_tiering_policy().data_placement.as_reference_name(), + self.policy.as_config_name(), + self.ssd_storage_engine.as_config_name(), + self.cache.production_tiering_policy().data_placement.as_config_name(), self.eviction_enabled(), stats.memory_bytes, stats.pmem_bytes, @@ -2977,6 +3024,8 @@ struct CacheOrderNode { key: CacheKey, prev: u32, next: u32, + access_prev: u32, + access_next: u32, } /// Recency ordering over cache keys, from least recently used at the front to @@ -2997,6 +3046,8 @@ pub struct CacheKeyOrder { index: HashMap, head: u32, tail: u32, + access_head: u32, + access_tail: u32, } impl Default for CacheKeyOrder { @@ -3013,6 +3064,8 @@ impl CacheKeyOrder { index: HashMap::new(), head: CACHE_ORDER_NIL, tail: CACHE_ORDER_NIL, + access_head: CACHE_ORDER_NIL, + access_tail: CACHE_ORDER_NIL, } } @@ -3049,10 +3102,13 @@ impl CacheKeyOrder { if let Some(&node) = self.index.get(&key) { self.unlink(node); self.link_back(node); + self.unlink_access(node); + self.link_access_back(node); return; } let node = self.alloc(key.clone()); self.link_back(node); + self.link_access_back(node); self.index.insert(key, node); } @@ -3068,6 +3124,7 @@ impl CacheKeyOrder { } let node = self.alloc(key.clone()); self.link_back(node); + self.link_access_back(node); self.index.insert(key, node); } @@ -3076,16 +3133,22 @@ impl CacheKeyOrder { if let Some(&node) = self.index.get(&key) { self.unlink(node); self.link_front(node); + self.unlink_access(node); + self.link_access_front(node); return; } let node = self.alloc(key.clone()); self.link_front(node); + self.link_access_front(node); self.index.insert(key, node); } - /// Record a hit: move `key` to the back if it is present. Returns whether - /// the key was there. - pub fn touch(&mut self, key: &CacheKey) -> bool { + /// Move `key` to the back of the insertion order if it is present. + /// Returns whether the key was there. + /// + /// This is not how a hit is recorded — a hit must leave the insertion + /// order alone, which is what `touch_access` is for. + pub fn move_to_back(&mut self, key: &CacheKey) -> bool { let Some(&node) = self.index.get(key) else { return false; }; @@ -3097,12 +3160,33 @@ impl CacheKeyOrder { true } + /// Record an access: move `key` to the back of the access order, leaving + /// the insertion order alone. Returns whether the key was there. + /// + /// The two orders answer different questions. Eviction that only ever + /// looks at the front of a list needs the front to hold entries nobody + /// wants; insertion order never moves an entry no matter how often it is + /// read, so a popular entry written early sits at the front forever and is + /// offered up on every pass. + pub fn touch_access(&mut self, key: &CacheKey) -> bool { + let Some(&node) = self.index.get(key) else { + return false; + }; + if node == self.access_tail { + return true; + } + self.unlink_access(node); + self.link_access_back(node); + true + } + /// Remove `key`, returning whether it was present. pub fn remove(&mut self, key: &CacheKey) -> bool { let Some(node) = self.index.remove(key) else { return false; }; self.unlink(node); + self.unlink_access(node); self.release(node); true } @@ -3115,6 +3199,7 @@ impl CacheKeyOrder { let node = self.head; let key = self.nodes[node as usize].key.clone(); self.unlink(node); + self.unlink_access(node); self.index.remove(&key); self.release(node); Some(key) @@ -3132,6 +3217,7 @@ impl CacheKeyOrder { let key = self.nodes[cursor as usize].key.clone(); self.index.remove(&key); self.unlink(cursor); + self.unlink_access(cursor); self.release(cursor); } cursor = next; @@ -3144,6 +3230,8 @@ impl CacheKeyOrder { self.index.clear(); self.head = CACHE_ORDER_NIL; self.tail = CACHE_ORDER_NIL; + self.access_head = CACHE_ORDER_NIL; + self.access_tail = CACHE_ORDER_NIL; } /// Iterate from most to least recently used. @@ -3162,18 +3250,30 @@ impl CacheKeyOrder { } } + /// Iterate the access order, least recently accessed first. + pub fn iter_access(&self) -> CacheKeyOrderAccessIter<'_> { + CacheKeyOrderAccessIter { + order: self, + cursor: self.access_head, + } + } + fn alloc(&mut self, key: CacheKey) -> u32 { if let Some(index) = self.free.pop() { let node = &mut self.nodes[index as usize]; node.key = key; node.prev = CACHE_ORDER_NIL; node.next = CACHE_ORDER_NIL; + node.access_prev = CACHE_ORDER_NIL; + node.access_next = CACHE_ORDER_NIL; return index; } self.nodes.push(CacheOrderNode { key, prev: CACHE_ORDER_NIL, next: CACHE_ORDER_NIL, + access_prev: CACHE_ORDER_NIL, + access_next: CACHE_ORDER_NIL, }); (self.nodes.len() - 1) as u32 } @@ -3206,6 +3306,47 @@ impl CacheKeyOrder { self.head = node; } + fn link_access_back(&mut self, node: u32) { + let old_tail = self.access_tail; + self.nodes[node as usize].access_prev = old_tail; + self.nodes[node as usize].access_next = CACHE_ORDER_NIL; + if old_tail == CACHE_ORDER_NIL { + self.access_head = node; + } else { + self.nodes[old_tail as usize].access_next = node; + } + self.access_tail = node; + } + + fn link_access_front(&mut self, node: u32) { + let old_head = self.access_head; + self.nodes[node as usize].access_next = old_head; + self.nodes[node as usize].access_prev = CACHE_ORDER_NIL; + if old_head == CACHE_ORDER_NIL { + self.access_tail = node; + } else { + self.nodes[old_head as usize].access_prev = node; + } + self.access_head = node; + } + + fn unlink_access(&mut self, node: u32) { + let prev = self.nodes[node as usize].access_prev; + let next = self.nodes[node as usize].access_next; + if prev == CACHE_ORDER_NIL { + self.access_head = next; + } else { + self.nodes[prev as usize].access_next = next; + } + if next == CACHE_ORDER_NIL { + self.access_tail = prev; + } else { + self.nodes[next as usize].access_prev = prev; + } + self.nodes[node as usize].access_prev = CACHE_ORDER_NIL; + self.nodes[node as usize].access_next = CACHE_ORDER_NIL; + } + fn unlink(&mut self, node: u32) { let prev = self.nodes[node as usize].prev; let next = self.nodes[node as usize].next; @@ -3244,6 +3385,24 @@ pub struct CacheKeyOrderRevIter<'a> { cursor: u32, } +pub struct CacheKeyOrderAccessIter<'a> { + order: &'a CacheKeyOrder, + cursor: u32, +} + +impl<'a> Iterator for CacheKeyOrderAccessIter<'a> { + type Item = &'a CacheKey; + + fn next(&mut self) -> Option { + if self.cursor == CACHE_ORDER_NIL { + return None; + } + let node = &self.order.nodes[self.cursor as usize]; + self.cursor = node.access_next; + Some(&node.key) + } +} + impl<'a> Iterator for CacheKeyOrderRevIter<'a> { type Item = &'a CacheKey; diff --git a/src/runtime/multilayer_cache.rs b/src/runtime/multilayer_cache.rs index 0c7e8e7..e1c85ea 100644 --- a/src/runtime/multilayer_cache.rs +++ b/src/runtime/multilayer_cache.rs @@ -71,21 +71,21 @@ pub trait CacheApi { fn capacity_cache(&self) -> usize; fn capacity_for_instance_cache(&self, instance_type: CacheInstanceType) -> usize { match instance_type { - CacheInstanceType::kUnified => self.capacity_cache(), + CacheInstanceType::Unified => self.capacity_cache(), _ => self.capacity_cache(), } } fn set_capacity_cache(&self, capacity: usize); fn set_capacity_for_instance_cache(&self, instance_type: CacheInstanceType, capacity: usize) { match instance_type { - CacheInstanceType::kUnified => self.set_capacity_cache(capacity), + CacheInstanceType::Unified => self.set_capacity_cache(capacity), _ => self.set_capacity_cache(capacity), } } fn size_cache(&self) -> usize; fn used_cache(&self, instance_type: CacheInstanceType) -> usize { match instance_type { - CacheInstanceType::kUnified => self.size_cache(), + CacheInstanceType::Unified => self.size_cache(), _ => self.size_cache(), } } @@ -180,28 +180,55 @@ struct SlotEvictionGroup { } impl SlotEvictionGroup { - fn new(victim: CacheKey, score: EvictionScore) -> Self { + fn new(victim: &CacheKey, score: EvictionScore) -> Self { Self { group_score: score, - victim, + victim: victim.clone(), victim_score: score, } } - fn observe(&mut self, key: CacheKey, score: EvictionScore) { + /// Fold one more member into the group. + /// + /// The key is borrowed and only cloned when it actually takes over as the + /// group's victim. Taking it by value cloned every key in the tier on + /// every eviction, and all but one of those clones was discarded. + fn observe(&mut self, key: &CacheKey, score: EvictionScore) { self.group_score.hotness = self.group_score.hotness.max(score.hotness); self.group_score.hits = self.group_score.hits.saturating_add(score.hits); self.group_score.last_access_epoch = self .group_score .last_access_epoch .max(score.last_access_epoch); - if score < self.victim_score || (score == self.victim_score && key < self.victim) { - self.victim = key; + if score < self.victim_score || (score == self.victim_score && *key < self.victim) { + self.victim = key.clone(); self.victim_score = score; } } } +/// How entries are grouped when picking an eviction victim. +/// +/// Entries that share a routing slot are weighed as a unit; everything else +/// stands alone. Borrowing the key's parts keeps this free of allocation: the +/// grouping used to render one `String` per resident entry per eviction, which +/// on a cache at capacity is a per-write cost proportional to the cache size. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum EvictionGroupKey<'a> { + Slot(u32), + Object(&'a str, &'a str), +} + +/// A chosen victim plus how many candidate groups were weighed to find it. +/// +/// The count feeds the `eviction_sampled_groups` statistic, which is what says +/// whether selection is inspecting a bounded number of candidates or the whole +/// tier. +struct PickedEvictionVictim { + victim: Option<(CacheKey, EvictionReason, u64)>, + groups_weighed: usize, +} + #[derive(Debug, Clone)] pub struct MultiLayerCache { inner: Arc>, @@ -232,12 +259,12 @@ struct CacheInner { memory: HashMap>, pmem: HashMap>, disk_index: HashMap, - disk_fifo_order: CacheKeyOrder, + disk_order: CacheKeyOrder, pinned: HashMap, pinned_handle_bytes: HashMap, pinned_removed_bytes: HashMap, - memory_fifo_order: CacheKeyOrder, - pmem_fifo_order: CacheKeyOrder, + memory_order: CacheKeyOrder, + pmem_order: CacheKeyOrder, async_writeback_queue: VecDeque, async_writeback_positions: HashMap, async_writeback_queue_bytes: u64, @@ -593,15 +620,15 @@ impl MultiLayerCache { ); cache.set_replacement_policy_for_tier( CacheTier::Memory, - CacheReplacementPolicy::from_reference_name(&options.cache_dram_replacement_policy), + CacheReplacementPolicy::from_config_name(&options.cache_dram_replacement_policy), ); cache.set_replacement_policy_for_tier( CacheTier::Pmem, - CacheReplacementPolicy::from_reference_name(&options.cache_pmem_replacement_policy), + CacheReplacementPolicy::from_config_name(&options.cache_pmem_replacement_policy), ); cache.set_replacement_policy_for_tier( CacheTier::Ssd, - CacheReplacementPolicy::from_reference_name(&options.cache_ssd_replacement_policy), + CacheReplacementPolicy::from_config_name(&options.cache_ssd_replacement_policy), ); cache.set_ssd_instance_only(options.cache_ssd_instance_only); cache.set_pmem_paths(options.pmem_paths); @@ -646,12 +673,12 @@ impl MultiLayerCache { memory: HashMap::new(), pmem: HashMap::new(), disk_index: HashMap::new(), - disk_fifo_order: CacheKeyOrder::new(), + disk_order: CacheKeyOrder::new(), pinned: HashMap::new(), pinned_handle_bytes: HashMap::new(), pinned_removed_bytes: HashMap::new(), - memory_fifo_order: CacheKeyOrder::new(), - pmem_fifo_order: CacheKeyOrder::new(), + memory_order: CacheKeyOrder::new(), + pmem_order: CacheKeyOrder::new(), async_writeback_queue: VecDeque::new(), async_writeback_positions: HashMap::new(), async_writeback_queue_bytes: 0, @@ -721,12 +748,27 @@ impl MultiLayerCache { self.inner.read().expect("cache lock poisoned").started } + /// Total capacity across the tiers, counted the way `size` counts usage. + /// + /// Side-by-side placement holds distinct keys in Dram and Pmem, so the two + /// capacities add. Tiered placement holds a given key in at most one of + /// them, so the pair contributes the larger of the two rather than their + /// sum. Summing under tiered placement overstates capacity, and since + /// `size` already takes the maximum there, it made a full cache report as + /// roughly half used. pub fn capacity(&self) -> usize { let inner = self.inner.read().expect("cache lock poisoned"); - inner - .memory_capacity_bytes - .saturating_add(inner.pmem_capacity_bytes) - .max(inner.ssd_capacity_bytes) + let volatile_bytes = if matches!( + inner.tiering_policy.data_placement, + CacheDataPlacement::SideBySide + ) { + inner + .memory_capacity_bytes + .saturating_add(inner.pmem_capacity_bytes) + } else { + inner.memory_capacity_bytes.max(inner.pmem_capacity_bytes) + }; + volatile_bytes.max(inner.ssd_capacity_bytes) } pub fn capacity_for_tier(&self, tier: CacheTier) -> usize { @@ -1218,8 +1260,10 @@ impl MultiLayerCache { if let Some(value) = inner.memory.get(key).cloned() { inner.stats.memory_hits += 1; inner.record_hit(key, value.len()); - inner.record_get_latency(started); - inner.record_read_through_latency(started); + // One interval, two histograms: read the clock once. + let micros = elapsed_micros(started); + inner.record_get_latency_micros(micros); + inner.record_read_through_latency_micros(micros); return Ok(Some(CacheReadResult { value: value.to_vec(), tier: CacheReadTier::Memory, @@ -2477,7 +2521,7 @@ impl MultiLayerCache { #[allow(non_snake_case)] pub fn SetDRAMPMEMDataPlacementType(&self, placement: DRAMPMEMDataPlacementType) { - self.set_reference_data_placement_type(placement); + self.set_config_data_placement_type(placement); } #[allow(non_snake_case)] @@ -2487,7 +2531,7 @@ impl MultiLayerCache { #[allow(non_snake_case)] pub fn GetDRAMPMEMDataPlacementType(&self) -> DRAMPMEMDataPlacementType { - self.reference_data_placement_type() + self.config_data_placement_type() } #[allow(non_snake_case)] @@ -2709,7 +2753,7 @@ impl MultiLayerCache { .data_placement } - pub fn reference_data_placement_type(&self) -> DRAMPMEMDataPlacementType { + pub fn config_data_placement_type(&self) -> DRAMPMEMDataPlacementType { self.data_placement().into() } @@ -2718,7 +2762,7 @@ impl MultiLayerCache { inner.tiering_policy.data_placement = placement; } - pub fn set_reference_data_placement_type(&self, placement: DRAMPMEMDataPlacementType) { + pub fn set_config_data_placement_type(&self, placement: DRAMPMEMDataPlacementType) { self.set_data_placement(placement.into()); } @@ -2748,8 +2792,8 @@ impl MultiLayerCache { if enabled { inner.memory.clear(); inner.pmem.clear(); - inner.memory_fifo_order.clear(); - inner.pmem_fifo_order.clear(); + inner.memory_order.clear(); + inner.pmem_order.clear(); inner.memory_bytes = 0; inner.pmem_bytes = 0; inner.refresh_usage_stats(); @@ -4816,7 +4860,7 @@ impl CacheInner { if let Some(old_len) = self.disk_index.insert(key.clone(), block_len as u64) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len as u64); self.record_metadata( &key, @@ -5028,10 +5072,10 @@ impl CacheInner { .collect::>(); if staged_keys.len() > SET_MEMBERSHIP_THRESHOLD { let staged_key_set = staged_keys.iter().cloned().collect::>(); - self.disk_fifo_order + self.disk_order .retain(|candidate| !staged_key_set.contains(candidate)); } else { - self.disk_fifo_order + self.disk_order .retain(|candidate| !staged_keys.contains(candidate)); } } @@ -5039,7 +5083,7 @@ impl CacheInner { if let Some(old_len) = self.disk_index.insert(entry.key.clone(), entry.block_len) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } - self.disk_fifo_order.push_back_if_absent(entry.key.clone()); + self.disk_order.push_back_if_absent(entry.key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(entry.block_len); self.record_metadata( &entry.key, @@ -5116,7 +5160,7 @@ impl CacheInner { if let Some(old_len) = self.disk_index.insert(key.clone(), block_len) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len); self.record_metadata( &key, @@ -5178,7 +5222,7 @@ impl CacheInner { if let Some(old_len) = self.disk_index.insert(key.clone(), block_len as u64) { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block_len as u64); self.record_metadata( &key, @@ -5208,7 +5252,7 @@ impl CacheInner { self.pinned_removed_bytes.insert(key.clone(), value.len()); } } - self.memory_fifo_order.remove(key); + self.memory_order.remove(key); } CacheTier::Pmem => { if let Some(value) = self.pmem.remove(key) { @@ -5217,7 +5261,7 @@ impl CacheInner { self.pinned_removed_bytes.insert(key.clone(), value.len()); } } - self.pmem_fifo_order.remove(key); + self.pmem_order.remove(key); self.persist_pmem_delete(key)?; } CacheTier::Ssd => { @@ -5225,7 +5269,7 @@ impl CacheInner { self.ssd_bytes = self.ssd_bytes.saturating_sub(old_len); } self.delete_ssd_block(key)?; - self.disk_fifo_order.remove(key); + self.disk_order.remove(key); self.stats.disk_bytes = self.ssd_bytes; self.append_disk_manifest_delete(key)?; } @@ -5326,20 +5370,20 @@ impl CacheInner { .remove(key) .unwrap_or(existing_block.len() as u64); self.ssd_bytes = self.ssd_bytes.saturating_sub(indexed_old_len); - self.disk_fifo_order.remove(key); + self.disk_order.remove(key); self.evict_ssd_for(block.len() as u64); if self.ssd_bytes.saturating_add(block.len() as u64) > self.ssd_capacity_bytes as u64 { self.disk_index.insert(key.clone(), indexed_old_len); - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(indexed_old_len); self.stats.disk_bytes = self.ssd_bytes; return Err(CacheError::CapacityExceeded); } self.write_ssd_block(key, &block)?; self.disk_index.insert(key.clone(), block.len() as u64); - self.disk_fifo_order.push_back_if_absent(key.clone()); + self.disk_order.push_back_if_absent(key.clone()); self.ssd_bytes = self.ssd_bytes.saturating_add(block.len() as u64); self.record_metadata( key, @@ -5410,9 +5454,9 @@ impl MultiLayerCache { } } inner - .memory_fifo_order + .memory_order .retain(|key| key.shard_id != shard_id); - inner.pmem_fifo_order.retain(|key| key.shard_id != shard_id); + inner.pmem_order.retain(|key| key.shard_id != shard_id); let disk_keys = inner .disk_index .keys() @@ -5425,7 +5469,7 @@ impl MultiLayerCache { disk_bytes_before.saturating_add(inner.disk_index.remove(key).unwrap_or_default()); } let _ = inner.delete_ssd_blocks(&disk_keys); - inner.disk_fifo_order.retain(|key| key.shard_id != shard_id); + inner.disk_order.retain(|key| key.shard_id != shard_id); inner.metadata.retain(|key, _| key.shard_id != shard_id); inner.pinned.retain(|key, _| key.shard_id != shard_id); inner @@ -5492,13 +5536,13 @@ impl MultiLayerCache { } let _ = inner.delete_ssd_blocks(&disk_delete_keys); inner - .memory_fifo_order + .memory_order .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); inner - .pmem_fifo_order + .pmem_order .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); inner - .disk_fifo_order + .disk_order .retain(|key| !(key.shard_id == shard_id && key.selector.starts_with(&prefix))); inner.stats.invalidations = inner .stats @@ -5559,13 +5603,13 @@ impl MultiLayerCache { inner.metadata.remove(key); } let _ = inner.delete_ssd_blocks(&disk_delete_keys); - inner.memory_fifo_order.retain(|key| { + inner.memory_order.retain(|key| { !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) }); - inner.pmem_fifo_order.retain(|key| { + inner.pmem_order.retain(|key| { !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) }); - inner.disk_fifo_order.retain(|key| { + inner.disk_order.retain(|key| { !(key.shard_id == shard_id && key.namespace == "page" && key.record_key == record_key) }); inner.stats.invalidations = inner diff --git a/src/runtime/replacement.rs b/src/runtime/replacement.rs index 20d6d56..2db82a9 100644 --- a/src/runtime/replacement.rs +++ b/src/runtime/replacement.rs @@ -240,7 +240,7 @@ pub enum CacheReplacementPolicy { } impl CacheReplacementPolicy { - pub fn from_reference_name(value: &str) -> Self { + pub fn from_config_name(value: &str) -> Self { if value.eq_ignore_ascii_case("fifo") { Self::Fifo } else if value.eq_ignore_ascii_case("slru") { @@ -250,7 +250,7 @@ impl CacheReplacementPolicy { } } - pub fn as_reference_name(self) -> &'static str { + pub fn as_config_name(self) -> &'static str { match self { Self::Fifo => "FIFO", Self::Slru => "SLRU", @@ -1028,6 +1028,13 @@ impl ReplacementArc { Ok(()) } + /// Drop every tracked key and put this policy back in the uninitialized + /// state. + /// + /// Unlike the Fifo and SLRU policies, resetting this one *does* require + /// another `init()` before it will accept keys. That asymmetry is + /// intentional and load-bearing for callers that drive the adaptive + /// lists directly; do not "harmonize" it away. pub fn reset(&mut self) -> Result<(), CacheError> { self.initialized = false; self.arc_list.reset(); @@ -1243,12 +1250,18 @@ impl ReplacementFIFO { self.base.init() } + /// Drop every tracked buffer and return the policy to an empty state. + /// + /// A successful reset leaves the policy *initialized and usable*: it + /// empties the index, not the policy's lifecycle. Clearing the + /// initialized flag here would make every later `put` return "no + /// evictions" while silently discarding the buffer, turning a reset + /// cache into a black hole that never reports an error. pub fn reset(&mut self) -> Result<(), CacheError> { self.index.clear(); self.queue.clear(); self.arena.clear(); self.base.used = 0; - self.base.initialized = false; Ok(()) } @@ -1485,7 +1498,7 @@ struct SlruEntry { node: u32, } -/// A hash-partitioned shard: three LRU lists plus the shard byte accounting. +/// A hash-partitioned shard: three Lru lists plus the shard byte accounting. #[derive(Debug)] struct SlruSegment { lists: [KeyList; SLRU_LIST_COUNT], @@ -1522,10 +1535,10 @@ enum SlruMaintainerStep { Evicted(CacheBuffer), } -/// Segmented LRU replacement policy. +/// Segmented Lru replacement policy. /// /// The cache is hash-partitioned into [`ReplacementSLRU::num_segments`] shards, -/// each holding an independent hot/warm/cold LRU triple with its own byte +/// each holding an independent hot/warm/cold Lru triple with its own byte /// budget of `capacity / num_segments`. Inserts, lookups and eviction only ever /// touch the shard owning the key, so eviction scans a small shard-local list /// instead of one global list, and no single hot list can consume the whole @@ -1585,6 +1598,11 @@ impl ReplacementSLRU { self.base.init() } + /// Drop every tracked buffer and return the policy to an empty state. + /// + /// As with the other policies, a successful reset leaves this one + /// initialized and ready to accept buffers again; see the note on + /// `ReplacementFIFO::reset`. pub fn reset(&mut self) -> Result<(), CacheError> { self.index.clear(); for segment in &mut self.segments { @@ -1595,7 +1613,6 @@ impl ReplacementSLRU { } self.arena.clear(); self.base.used = 0; - self.base.initialized = false; Ok(()) } @@ -1665,7 +1682,7 @@ impl ReplacementSLRU { /// /// Only the access flag moves: an entry is not re-fronted in its list on a /// read. List position is decided by the maintainer and by eviction, which - /// is what keeps this a segmented policy rather than a plain LRU — reading + /// is what keeps this a segmented policy rather than a plain Lru — reading /// a key once does not jump it ahead of one that was read twice. pub fn get(&mut self, key: &str) -> Option { let entry = self.index.get_mut(key)?; @@ -2243,7 +2260,7 @@ impl ReplacementSLRU { } } -/// A segmented LRU that can be shared across threads, holding one lock per +/// A segmented Lru that can be shared across threads, holding one lock per /// segment instead of one lock over the whole policy. /// /// [`ReplacementSLRU`] partitions its lists into segments, but it is driven diff --git a/src/runtime/storage_engines.rs b/src/runtime/storage_engines.rs index 3b8a627..9f6b9d2 100644 --- a/src/runtime/storage_engines.rs +++ b/src/runtime/storage_engines.rs @@ -81,7 +81,7 @@ impl CacheOptions { } pub fn with_replacement_policy(mut self, policy: CacheReplacementPolicy) -> Self { - let name = policy.as_reference_name().to_string(); + let name = policy.as_config_name().to_string(); self.cache_dram_replacement_policy = name.clone(); self.cache_pmem_replacement_policy = name.clone(); self.cache_ssd_replacement_policy = name; @@ -93,7 +93,7 @@ impl CacheOptions { tier: CacheTier, policy: CacheReplacementPolicy, ) -> Self { - let name = policy.as_reference_name().to_string(); + let name = policy.as_config_name().to_string(); match tier { CacheTier::Memory => self.cache_dram_replacement_policy = name, CacheTier::Pmem => self.cache_pmem_replacement_policy = name, @@ -108,12 +108,12 @@ impl CacheOptions { placement: CacheDataPlacement, threshold: usize, ) -> Self { - self.cache_dram_pmem_data_placement_type = placement.as_reference_name().to_string(); + self.cache_dram_pmem_data_placement_type = placement.as_config_name().to_string(); self.cache_dram_pmem_data_placement_threshold = threshold; self } - pub fn with_reference_dram_pmem_data_placement( + pub fn with_config_dram_pmem_data_placement( self, placement: DRAMPMEMDataPlacementType, threshold: usize, @@ -127,7 +127,7 @@ impl CacheOptions { placement: DRAMPMEMDataPlacementType, threshold: usize, ) -> Self { - self.with_reference_dram_pmem_data_placement(placement, threshold) + self.with_config_dram_pmem_data_placement(placement, threshold) } pub fn with_metric_id_prefix(mut self, prefix: impl Into) -> Self { @@ -171,7 +171,7 @@ impl CacheOptions { memory_capacity_bytes: self.dram_capacity, pmem_capacity_bytes: self.pmem_capacity, ssd_capacity_bytes: self.ssd_capacity, - data_placement: CacheDataPlacement::from_reference_name( + data_placement: CacheDataPlacement::from_config_name( &self.cache_dram_pmem_data_placement_type, ), data_placement_threshold_bytes: self.cache_dram_pmem_data_placement_threshold, @@ -1401,7 +1401,7 @@ impl PmemAllocatorRecoverListenerImpl { continue; }; let buffer = - MemStorage::create_cache_buffer_from_data(record, StorageEngineType::kPMEM, true)?; + MemStorage::create_cache_buffer_from_data(record, StorageEngineType::Pmem, true)?; callback.on_recover_data(&key, buffer); valid_records = valid_records.saturating_add(1); } @@ -1464,7 +1464,7 @@ impl StorageEngineSimple { } fn buffer_from_record(&self, record: &[u8]) -> Result { - MemStorage::create_cache_buffer_from_data(record, StorageEngineType::kSimple, false) + MemStorage::create_cache_buffer_from_data(record, StorageEngineType::Simple, false) } pub fn test_get_num_delete_completed_count(&self) -> u32 { @@ -1634,7 +1634,7 @@ impl StorageEngineApi for StorageEngineSimple { } fn storage_engine_type(&self) -> StorageEngineType { - StorageEngineType::kSimple + StorageEngineType::Simple } } @@ -2389,7 +2389,7 @@ impl StorageEngineApi for StorageEngineRocksDB { } fn storage_engine_type(&self) -> StorageEngineType { - StorageEngineType::kSSD + StorageEngineType::Ssd } } @@ -2475,7 +2475,7 @@ pub struct StorageEngineMultiSSD { impl StorageEngineMultiSSD { pub fn new(paths: impl IntoIterator, capacity: u64) -> Self { - Self::with_type(paths, capacity, StorageEngineType::kSSD) + Self::with_type(paths, capacity, StorageEngineType::Ssd) } pub fn with_paths(paths: impl IntoIterator, capacity: u64) -> Self { @@ -2757,7 +2757,7 @@ impl StorageEngineApi for StorageEngineMultiSSD { } fn storage_engine_type(&self) -> StorageEngineType { - StorageEngineType::kMultiSSD + StorageEngineType::MultiSsd } } @@ -3036,7 +3036,7 @@ impl PMemDispatcher { }) .collect(); Self { - alloc_type: AllocatorType::kLogBasedAllocator, + alloc_type: AllocatorType::LogBasedAllocator, writers, current_numa: 0, stopped: true, @@ -3046,7 +3046,7 @@ impl PMemDispatcher { pub fn from_allocators(allocators: Vec) -> Self { let writers = allocators.into_iter().map(AsyncWriter::new).collect(); Self { - alloc_type: AllocatorType::kLogBasedAllocator, + alloc_type: AllocatorType::LogBasedAllocator, writers, current_numa: 0, stopped: true, diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 638284e..ad9ca87 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -6,7 +6,7 @@ mod tests { use super::*; #[test] - fn parity_rdma_response_hash_table_and_index_surface_round_trip() { + fn rdma_response_hash_table_and_index_surface_round_trip() { let mut response = RDMAResponse::New(32); let first_allocation = response.allocation_addr(); assert_eq!(response.GetRespSize(), 32); @@ -44,9 +44,9 @@ mod tests { entry.set_signature_96(sig96); entry.SetDataLength(128); entry.SetVersion(); - entry.set_packed_addr(0x1234, RdmaStorageEngineType::PMEM, 128); + entry.set_packed_addr(0x1234, RdmaStorageEngineType::Pmem, 128); assert_eq!(entry.GetPtr(), 0x1234); - assert_eq!(entry.GetType(), RdmaStorageEngineType::PMEM.as_code()); + assert_eq!(entry.GetType(), RdmaStorageEngineType::Pmem.as_code()); assert_eq!(entry.GetOverflowFlag(), 0); assert_eq!(entry.GetLength(), 128); assert_eq!(entry.GetVersion(), 0); @@ -56,9 +56,9 @@ mod tests { overflow.set_signature_128(sig128); overflow.SetDataLength(i32::MAX); overflow.SetVersion(); - overflow.set_packed_addr(0x2222, RdmaStorageEngineType::SSD, RDMA_MAX_BLOCK_SIZE + 1); + overflow.set_packed_addr(0x2222, RdmaStorageEngineType::Ssd, RDMA_MAX_BLOCK_SIZE + 1); assert_eq!(overflow.GetPtr(), 0x2222); - assert_eq!(overflow.GetType(), RdmaStorageEngineType::SSD.as_code()); + assert_eq!(overflow.GetType(), RdmaStorageEngineType::Ssd.as_code()); assert_eq!(overflow.GetOverflowFlag(), 1); assert_eq!(overflow.GetSignature128b(), sig128); @@ -67,7 +67,7 @@ mod tests { assert_eq!(table.GetNumEntries(), 0); assert!(table.AllBucketsUnlocked()); - let put = table.Put(key.clone(), 0x1000, 11, RdmaStorageEngineType::DRAM); + let put = table.Put(key.clone(), 0x1000, 11, RdmaStorageEngineType::Dram); assert_eq!(put.status, RDMA_OP_SUCCESS); assert_eq!(put.old_addr, None); assert_eq!(table.GetNumEntries(), 1); @@ -75,31 +75,31 @@ mod tests { let got = table.Get(&key); assert_eq!(got.addr, Some(0x1000)); assert_eq!(got.len, 11 + RDMA_DATA_HEADER + RDMA_CRC_LEN); - assert_eq!(got.storage_type, RdmaStorageEngineType::DRAM); + assert_eq!(got.storage_type, RdmaStorageEngineType::Dram); - let update = table.Put(key.clone(), 0x2000, 17, RdmaStorageEngineType::SSD); + let update = table.Put(key.clone(), 0x2000, 17, RdmaStorageEngineType::Ssd); assert_eq!(update.status, RDMA_OP_SUCCESS); assert_eq!(update.old_addr, Some(0x1000)); assert_eq!(update.old_len, 11 + RDMA_DATA_HEADER + RDMA_CRC_LEN); - assert_eq!(update.old_type, RdmaStorageEngineType::DRAM); + assert_eq!(update.old_type, RdmaStorageEngineType::Dram); assert_eq!(table.GetNumEntries(), 1); let got = table.Get(&key); assert_eq!(got.addr, Some(0x2000)); - assert_eq!(got.storage_type, RdmaStorageEngineType::SSD); + assert_eq!(got.storage_type, RdmaStorageEngineType::Ssd); let del = table.Del(&key); assert_eq!(del.status, RDMA_OP_SUCCESS); assert_eq!(del.addr, Some(0x2000)); - assert_eq!(del.storage_type, RdmaStorageEngineType::SSD); + assert_eq!(del.storage_type, RdmaStorageEngineType::Ssd); assert_eq!(table.GetNumEntries(), 0); - assert_eq!(table.Get(&key).storage_type, RdmaStorageEngineType::INVALID); + assert_eq!(table.Get(&key).storage_type, RdmaStorageEngineType::Invalid); assert_eq!(table.Del(&key).status, RDMA_NOT_FOUND); assert!(table.AllBucketsUnlocked()); } #[test] - fn parity_rdma_std_allocator_allocates_and_frees_virtual_regions() { + fn rdma_std_allocator_allocates_and_frees_virtual_regions() { fn round_trip(allocator: &mut A) -> AllocatorPtr { let addr = allocator.allocate(64).expect("allocator ptr"); allocator.free(addr, 64); @@ -123,7 +123,7 @@ mod tests { } #[test] - fn parity_rdma_dram_and_pmem_storage_engines_round_trip_blocks() { + fn rdma_dram_and_pmem_storage_engines_round_trip_blocks() { let mut dram = RdmaStorageEngineDram::with_capacity(1024); let key = 1_i32.to_le_bytes(); let value = 7_i32.to_le_bytes(); @@ -173,25 +173,25 @@ mod tests { } #[test] - fn parity_rdma_cache_composes_index_storage_and_replacement_policy() { + fn rdma_cache_composes_index_storage_and_replacement_policy() { let key = 1_i32.to_le_bytes(); let value_one = 1_i32.to_le_bytes(); let value_two = 2_i32.to_le_bytes(); - let mut cache = RDMACache::new(1024, 1024, 1024, RdmaReplacementPolicyType::FIFO); - assert_eq!(cache.GetCapacity(RdmaStorageEngineType::DRAM), 1024); + let mut cache = RDMACache::new(1024, 1024, 1024, RdmaReplacementPolicyType::Fifo); + assert_eq!(cache.GetCapacity(RdmaStorageEngineType::Dram), 1024); assert_eq!( cache.GetReplacementPolicyType(), - RdmaReplacementPolicyType::FIFO + RdmaReplacementPolicyType::Fifo ); - cache.SetReplacementPolicy(RdmaReplacementPolicyType::LRU); + cache.SetReplacementPolicy(RdmaReplacementPolicyType::Lru); assert_eq!( cache.GetReplacementPolicyType(), - RdmaReplacementPolicyType::LRU + RdmaReplacementPolicyType::Lru ); assert_eq!( - RdmaReplacementPolicyType::LRU.as_replacement_policy_type(), - ReplacementPolicyType::kLRU + RdmaReplacementPolicyType::Lru.as_replacement_policy_type(), + ReplacementPolicyType::Lru ); let mut response = RDMAResponse::new(); @@ -200,7 +200,7 @@ mod tests { assert_eq!(response.GetResponse(), value_one); assert_eq!(cache.num_index_entries(), 1); assert_eq!( - cache.storage_stats(RdmaStorageEngineType::DRAM).unwrap().2, + cache.storage_stats(RdmaStorageEngineType::Dram).unwrap().2, 1 ); @@ -210,7 +210,7 @@ mod tests { assert_eq!(response.GetResponse(), value_two); assert_eq!(cache.num_index_entries(), 1); assert_eq!( - cache.storage_stats(RdmaStorageEngineType::DRAM).unwrap().2, + cache.storage_stats(RdmaStorageEngineType::Dram).unwrap().2, 1 ); @@ -222,7 +222,7 @@ mod tests { let pmem_key = b"pmem-key"; assert_eq!( - cache.InsertToStorage(RdmaStorageEngineType::PMEM, pmem_key, b"pmem-value"), + cache.InsertToStorage(RdmaStorageEngineType::Pmem, pmem_key, b"pmem-value"), RDMA_OP_SUCCESS ); response.Clear(); @@ -231,7 +231,7 @@ mod tests { let ssd_key = b"ssd-key"; assert_eq!( - cache.InsertToStorage(RdmaStorageEngineType::SSD, ssd_key, b"ssd-value"), + cache.InsertToStorage(RdmaStorageEngineType::Ssd, ssd_key, b"ssd-value"), RDMA_OP_SUCCESS ); response.Clear(); @@ -241,12 +241,12 @@ mod tests { let mut dram_only = RDMACache::with_dram_capacity(8); assert_eq!(dram_only.Insert(b"too-large", b"value"), RDMA_FAIL_ALLOC); assert_eq!( - dram_only.InsertToStorage(RdmaStorageEngineType::PMEM, b"k", b"v"), + dram_only.InsertToStorage(RdmaStorageEngineType::Pmem, b"k", b"v"), RDMA_FAIL_ALLOC ); - dram_only.InitStorageEngine(RdmaStorageEngineType::PMEM, 128); + dram_only.InitStorageEngine(RdmaStorageEngineType::Pmem, 128); assert_eq!( - dram_only.InsertToStorage(RdmaStorageEngineType::PMEM, b"k", b"v"), + dram_only.InsertToStorage(RdmaStorageEngineType::Pmem, b"k", b"v"), RDMA_OP_SUCCESS ); } @@ -301,7 +301,40 @@ mod tests { } #[test] - fn parity_unified_size_is_placement_aware() { + fn unified_capacity_is_placement_aware() { + let dir = tempfile::tempdir().unwrap(); + let cache = MultiLayerCache::with_tiering_policy( + dir.path(), + CacheTieringPolicy { + memory_capacity_bytes: 64, + pmem_capacity_bytes: 64, + ssd_capacity_bytes: 16, + data_placement: CacheDataPlacement::Tiered, + data_placement_threshold_bytes: 4, + memory_hotness_threshold: 0, + pmem_admit_hotness_threshold: 0, + ssd_admit_hotness_threshold: u32::MAX, + max_memory_block_bytes: 64, + max_pmem_block_bytes: 64, + max_ssd_block_bytes: 16, + ssd_write_through: false, + }, + CacheBlockOptions::default(), + ); + + // Tiered placement holds a key in at most one of the volatile tiers, + // so the pair contributes the larger of the two. This is the same rule + // Size already applies, and summing here would report a full cache as + // half used. + assert_eq!(cache.Capacity(), 64); + + // Side by side holds distinct keys in each tier, so they add. + cache.SetDataPlacementType(CacheDataPlacement::SideBySide); + assert_eq!(cache.Capacity(), 128); + } + + #[test] + fn unified_size_is_placement_aware() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -324,11 +357,11 @@ mod tests { let memory_key = CacheKey::string(7, "memory-size"); let pmem_key = CacheKey::string(7, "pmem-size"); cache - .TEST_Insert(CacheInstanceType::kDRAM, memory_key, b"abcd".to_vec(), 4) + .TEST_Insert(CacheInstanceType::Dram, memory_key, b"abcd".to_vec(), 4) .unwrap(); cache .TEST_Insert( - CacheInstanceType::kPMEM, + CacheInstanceType::Pmem, pmem_key, b"0123456789".to_vec(), 10, @@ -344,7 +377,7 @@ mod tests { } #[test] - fn parity_cache_api_aliases_match_insert_lookup_remove_semantics() { + fn cache_api_aliases_match_insert_lookup_remove_semantics() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(32, dir.path()); let key = CacheKey::string(31, "legacy-api"); @@ -791,7 +824,7 @@ mod tests { } #[test] - fn parity_simple_lru_cache_wrapper_evicts_like_public_stub_cache() { + fn simple_lru_cache_wrapper_evicts_like_public_stub_cache() { let cache = MatrixCacheBuilder::BuildSimpleLRUCache(12); assert!(cache.Stop()); assert!(cache.Start()); @@ -837,7 +870,7 @@ mod tests { } #[test] - fn parity_zero_copy_simple_lru_cache_keeps_removed_pinned_value_readable() { + fn zero_copy_simple_lru_cache_keeps_removed_pinned_value_readable() { let cache = MatrixCacheBuilder::BuildZeroCopySimpleLRUCache(8); let pinned_key = CacheKey::string(33, "pinned"); let cold_key = CacheKey::string(33, "cold"); @@ -899,7 +932,7 @@ mod tests { } #[test] - fn parity_string_cache_wrappers_match_tool_cache_interface() { + fn string_cache_wrappers_match_tool_cache_interface() { let simple = MatrixCacheBuilder::BuildConcurrentSimpleLRUCache(16); assert!(simple.Stop()); assert!(simple.Start()); @@ -930,16 +963,16 @@ mod tests { simple.RemoveAll().unwrap(); assert_eq!(simple.Size(), 0); - let exact_reference_name = ConcurrentSimpleLRUCache::new(32); - exact_reference_name + let exact_config_name = ConcurrentSimpleLRUCache::new(32); + exact_config_name .InsertDefaultSize("gamma", "three".to_string()) .unwrap(); assert_eq!( - exact_reference_name.Lookup("gamma").unwrap(), + exact_config_name.Lookup("gamma").unwrap(), Some("three".to_string()) ); - let string_api: &dyn StringCacheApi = &exact_reference_name; + let string_api: &dyn StringCacheApi = &exact_config_name; string_api .insert_string_default_size("delta", "four".to_string()) .unwrap(); @@ -950,7 +983,7 @@ mod tests { } #[test] - fn parity_memcached_wrapper_matches_tool_cache_surface_without_external_daemon() { + fn memcached_wrapper_matches_tool_cache_surface_without_external_daemon() { let cache = MatrixCacheBuilder::BuildMemcachedWrapper(8); assert_eq!(cache.configured_capacity(), 8); assert_eq!(cache.Capacity(), 8); @@ -1001,7 +1034,7 @@ mod tests { } #[test] - fn parity_multi_tier_string_cache_wraps_zero_copy_cache() { + fn multi_tier_string_cache_wraps_zero_copy_cache() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildMultiTierStringCache(CacheOptions { dram_capacity: 8, @@ -1034,7 +1067,7 @@ mod tests { } #[test] - fn parity_pascal_case_cache_methods_match_matrixcache_interface() { + fn pascal_case_cache_methods_match_matrixcache_interface() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache(CacheOptions { dram_capacity: 64, @@ -1109,7 +1142,7 @@ mod tests { } #[test] - fn parity_instance_controls_match_unified_cache_getters_and_setters() { + fn instance_controls_match_unified_cache_getters_and_setters() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -1130,26 +1163,26 @@ mod tests { CacheBlockOptions::default(), ); - assert_eq!(cache.GetCapacity(CacheInstanceType::kDRAM), 16); - assert_eq!(cache.GetCapacity(CacheInstanceType::kPMEM), 32); - assert_eq!(cache.GetCapacity(CacheInstanceType::kSSD), 128); - assert_eq!(cache.GetCapacity(CacheInstanceType::kUnified), 128); + assert_eq!(cache.GetCapacity(CacheInstanceType::Dram), 16); + assert_eq!(cache.GetCapacity(CacheInstanceType::Pmem), 32); + assert_eq!(cache.GetCapacity(CacheInstanceType::Ssd), 128); + assert_eq!(cache.GetCapacity(CacheInstanceType::Unified), 128); - cache.SetCapacityForInstance(CacheInstanceType::kDRAM, 8); - cache.SetCapacityForInstance(CacheInstanceType::kPMEM, 24); - cache.SetCapacityForInstance(CacheInstanceType::kSSD, 96); - assert_eq!(cache.get_capacity(CacheInstanceType::kDRAM), 8); - assert_eq!(cache.get_capacity(CacheInstanceType::kPMEM), 24); - assert_eq!(cache.get_capacity(CacheInstanceType::kSSD), 96); + cache.SetCapacityForInstance(CacheInstanceType::Dram, 8); + cache.SetCapacityForInstance(CacheInstanceType::Pmem, 24); + cache.SetCapacityForInstance(CacheInstanceType::Ssd, 96); + assert_eq!(cache.get_capacity(CacheInstanceType::Dram), 8); + assert_eq!(cache.get_capacity(CacheInstanceType::Pmem), 24); + assert_eq!(cache.get_capacity(CacheInstanceType::Ssd), 96); - cache.SetReplacementPolicyType(CacheInstanceType::kDRAM, CacheReplacementPolicy::Fifo); - cache.SetReplacementPolicyType(CacheInstanceType::kPMEM, CacheReplacementPolicy::Slru); + cache.SetReplacementPolicyType(CacheInstanceType::Dram, CacheReplacementPolicy::Fifo); + cache.SetReplacementPolicyType(CacheInstanceType::Pmem, CacheReplacementPolicy::Slru); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::Fifo ); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kPMEM), + cache.GetReplacementPolicyType(CacheInstanceType::Pmem), CacheReplacementPolicy::Slru ); @@ -1162,31 +1195,31 @@ mod tests { cache .Insert(memory_key.clone(), b"abcd".to_vec(), b"abcd".len()) .unwrap(); - assert!(cache.GetUsed(CacheInstanceType::kDRAM) > 0); + assert!(cache.GetUsed(CacheInstanceType::Dram) > 0); assert!(cache.Size() >= b"abcd".len()); } #[test] - fn parity_allocator_types_and_stats_match_cache_instance_storage_surface() { + fn allocator_types_and_stats_match_cache_instance_storage_surface() { assert_eq!( - AllocatorType::from_reference_name("kLogBasedAllocator"), - AllocatorType::kLogBasedAllocator + AllocatorType::from_config_name("kLogBasedAllocator"), + AllocatorType::LogBasedAllocator ); assert_eq!( - AllocatorType::from_reference_name("pool_based"), - AllocatorType::kPoolBasedAllocator + AllocatorType::from_config_name("pool_based"), + AllocatorType::PoolBasedAllocator ); - assert_eq!(AllocatorType::kJeAllocator.as_reference_name(), "JeAllocator"); + assert_eq!(AllocatorType::JeAllocator.as_config_name(), "JeAllocator"); let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); assert_eq!( instance.GetAllocatorType(), - AllocatorType::kPoolBasedAllocator + AllocatorType::PoolBasedAllocator ); assert_eq!(instance.GetAllocatorStats(), AllocatorStats::default()); @@ -1206,11 +1239,11 @@ mod tests { } #[test] - fn parity_cache_instance_latency_summary_uses_live_cache_metrics() { + fn cache_instance_latency_summary_uses_live_cache_metrics() { let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); instance.Put("latency-a", b"abc".to_vec()).unwrap(); @@ -1230,7 +1263,7 @@ mod tests { } #[test] - fn parity_allocator_metadata_structs_preserve_chunk_state() { + fn allocator_metadata_structs_preserve_chunk_state() { let stats = AllocatorStats::new(128, 32); assert_eq!(stats.NumAllocatedBytes(), 128); assert_eq!(stats.NumFreedBytes(), 32); @@ -1255,20 +1288,20 @@ mod tests { } #[test] - fn parity_allocator_recovery_surface_matches_pmem_and_pool_headers() { - assert_eq!(AllocatorType::kLogBasedAllocator as u8, 0); - assert_eq!(AllocatorType::kPoolBasedAllocator as u8, 1); - assert_eq!(AllocatorType::kJeAllocator as u8, 2); - assert_eq!(AllocatorType::kMaxCode as u8, 3); + fn allocator_recovery_surface_matches_pmem_and_pool_headers() { + assert_eq!(AllocatorType::LogBasedAllocator as u8, 0); + assert_eq!(AllocatorType::PoolBasedAllocator as u8, 1); + assert_eq!(AllocatorType::JeAllocator as u8, 2); + assert_eq!(AllocatorType::MaxCode as u8, 3); - assert_eq!(FlushPolicy::kNoFlush as u8, 0); - assert_eq!(FlushPolicy::kInstantFlush as u8, 1); - assert_eq!(FlushPolicy::kMiniBatchFlush as u8, 2); + assert_eq!(FlushPolicy::NoFlush as u8, 0); + assert_eq!(FlushPolicy::InstantFlush as u8, 1); + assert_eq!(FlushPolicy::MiniBatchFlush as u8, 2); assert_eq!( - FlushPolicy::from_reference_name("kInstantFlush"), - FlushPolicy::kInstantFlush + FlushPolicy::from_config_name("kInstantFlush"), + FlushPolicy::InstantFlush ); - assert_eq!(FlushPolicy::kMiniBatchFlush.as_reference_name(), "MiniBatchFlush"); + assert_eq!(FlushPolicy::MiniBatchFlush.as_config_name(), "MiniBatchFlush"); let mut recover = PmemRecoverStats::default(); recover.AddChunkStats(ChunkRecoverStats { @@ -1312,7 +1345,7 @@ mod tests { } #[test] - fn parity_specialized_allocator_aliases_share_common_allocator_surface() { + fn specialized_allocator_aliases_share_common_allocator_surface() { let mut je = JeAllocator::with_capacity(32); let ptr = je.Allocate(8).unwrap(); assert!(je.Contains(ptr)); @@ -1334,7 +1367,7 @@ mod tests { } #[test] - fn parity_je_allocator_enforces_capacity_and_tracks_stats() { + fn je_allocator_enforces_capacity_and_tracks_stats() { let mut allocator = JeAllocator::with_capacity(4 * 1024); let ptr = allocator.Allocate(1024).unwrap(); assert!(allocator.Contains(ptr)); @@ -1357,7 +1390,7 @@ mod tests { } #[test] - fn parity_pool_allocator_reuses_fixed_objects_and_tracks_chunks() { + fn pool_allocator_reuses_fixed_objects_and_tracks_chunks() { let mut allocator = PoolBasedMemoryAllocatorDram::new( 1 << 28, PoolBasedMemoryAllocatorBase::DEFAULT_MAX_THREAD_NUM, @@ -1386,10 +1419,10 @@ mod tests { } #[test] - fn parity_pool_allocator_rebalance_exposes_global_free_list_size() { + fn pool_allocator_rebalance_exposes_global_free_list_size() { let mut allocator = PoolBasedMemoryAllocatorPMem::pmem( "/tmp", - FlushPolicy::kNoFlush, + FlushPolicy::NoFlush, 1 << 28, PoolBasedMemoryAllocatorBase::DEFAULT_MAX_THREAD_NUM, PoolBasedMemoryAllocatorBase::DEFAULT_OBJECT_LEN, @@ -1415,7 +1448,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_supports_insert_assign_find_and_erase() { + fn concurrent_hash_map_supports_insert_assign_find_and_erase() { let map = ConcurrentHashMap::::new(2, 4); assert!(map.Empty()); assert_eq!(map.Size(), 0); @@ -1447,7 +1480,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_honors_capacity_and_shared_clones() { + fn concurrent_hash_map_honors_capacity_and_shared_clones() { let map = ConcurrentHashMap::::new(1, 1); assert!(map.Insert(7, "seven".to_string()).unwrap()); assert!(matches!( @@ -1470,7 +1503,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_exposes_at_iterate_and_emplace_surface() { + fn concurrent_hash_map_exposes_at_iterate_and_emplace_surface() { let map = ConcurrentHashMap::::new(2, 16); assert_eq!(map.At(&20), 0); assert_eq!(map.GetOrDefault(&20), 0); @@ -1497,7 +1530,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_erases_by_entry_and_predicate() { + fn concurrent_hash_map_erases_by_entry_and_predicate() { let map = ConcurrentHashMap::::new(3, 0); assert!(map.Insert("live".to_string(), 10).unwrap()); assert!(map.Insert("stale".to_string(), 20).unwrap()); @@ -1520,7 +1553,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_returns_iterator_style_insert_results() { + fn concurrent_hash_map_returns_iterator_style_insert_results() { let map = ConcurrentHashMap::::new(2, 4); let first = map.InsertEntry(1, 10).unwrap(); assert!(first.second); @@ -1552,7 +1585,7 @@ mod tests { } #[test] - fn parity_concurrent_hash_map_erases_entries_by_snapshot_predicate() { + fn concurrent_hash_map_erases_entries_by_snapshot_predicate() { let map = ConcurrentHashMap::::new(2, 0); for key in 0..10 { assert!(map.Insert(key, key).unwrap()); @@ -1566,7 +1599,7 @@ mod tests { } #[test] - fn parity_hist_stats_reports_percentiles_average_max_and_reset() { + fn hist_stats_reports_percentiles_average_max_and_reset() { let mut stats = HistStats::with_bucket_size(8); for value in [1, 2, 2, 4, 9] { stats.Append(value); @@ -1586,7 +1619,7 @@ mod tests { } #[test] - fn parity_hist_stats_merge_preserves_large_latency_tail() { + fn hist_stats_merge_preserves_large_latency_tail() { let mut left = HistStats::with_bucket_size(4); let mut right = HistStats::with_bucket_size(4); left.Append(1); @@ -1603,7 +1636,7 @@ mod tests { } #[test] - fn parity_test_instance_helpers_target_exact_cache_tiers() { + fn test_instance_helpers_target_exact_cache_tiers() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -1632,7 +1665,7 @@ mod tests { cache .TEST_Insert( - CacheInstanceType::kDRAM, + CacheInstanceType::Dram, memory_key.clone(), b"dram".to_vec(), b"dram".len(), @@ -1640,7 +1673,7 @@ mod tests { .unwrap(); cache .TEST_Insert( - CacheInstanceType::kPMEM, + CacheInstanceType::Pmem, pmem_key.clone(), b"pmem".to_vec(), b"pmem".len(), @@ -1648,7 +1681,7 @@ mod tests { .unwrap(); cache .TEST_Insert( - CacheInstanceType::kSSD, + CacheInstanceType::Ssd, ssd_key.clone(), b"ssd".to_vec(), b"ssd".len(), @@ -1656,7 +1689,7 @@ mod tests { .unwrap(); let memory_handle = cache - .TEST_Acquire(CacheInstanceType::kDRAM, &memory_key) + .TEST_Acquire(CacheInstanceType::Dram, &memory_key) .unwrap() .expect("memory handle"); assert_eq!(memory_handle.tier(), CacheReadTier::Memory); @@ -1664,7 +1697,7 @@ mod tests { cache.Release(memory_handle); let pmem_handle = cache - .TEST_Acquire(CacheInstanceType::kPMEM, &pmem_key) + .TEST_Acquire(CacheInstanceType::Pmem, &pmem_key) .unwrap() .expect("pmem handle"); assert_eq!(pmem_handle.tier(), CacheReadTier::Pmem); @@ -1675,7 +1708,7 @@ mod tests { assert_eq!(cache.stats().pinned_bytes, 0); let ssd_handle = cache - .TEST_Acquire(CacheInstanceType::kSSD, &ssd_key) + .TEST_Acquire(CacheInstanceType::Ssd, &ssd_key) .unwrap() .expect("ssd handle"); assert_eq!(ssd_handle.tier(), CacheReadTier::Ssd); @@ -1686,44 +1719,44 @@ mod tests { assert_eq!(cache.stats().pinned_bytes, 0); assert!(cache - .TEST_Acquire(CacheInstanceType::kPMEM, &memory_key) + .TEST_Acquire(CacheInstanceType::Pmem, &memory_key) .unwrap() .is_none()); assert!(cache - .TEST_Acquire(CacheInstanceType::kDRAM, &pmem_key) + .TEST_Acquire(CacheInstanceType::Dram, &pmem_key) .unwrap() .is_none()); cache - .TEST_Remove(CacheInstanceType::kPMEM, &pmem_key) + .TEST_Remove(CacheInstanceType::Pmem, &pmem_key) .unwrap(); assert!(cache - .TEST_Acquire(CacheInstanceType::kPMEM, &pmem_key) + .TEST_Acquire(CacheInstanceType::Pmem, &pmem_key) .unwrap() .is_none()); assert_eq!(cache.Lookup(&pmem_key).unwrap(), None); cache - .TEST_Remove(CacheInstanceType::kSSD, &ssd_key) + .TEST_Remove(CacheInstanceType::Ssd, &ssd_key) .unwrap(); assert!(cache - .TEST_Acquire(CacheInstanceType::kSSD, &ssd_key) + .TEST_Acquire(CacheInstanceType::Ssd, &ssd_key) .unwrap() .is_none()); assert!(matches!( cache.TEST_Insert( - CacheInstanceType::kUnified, + CacheInstanceType::Unified, CacheKey::string(44, "bad"), b"bad".to_vec(), 3, ), - Err(CacheError::UnsupportedInstance(CacheInstanceType::kUnified)) + Err(CacheError::UnsupportedInstance(CacheInstanceType::Unified)) )); } #[test] - fn parity_test_counter_and_path_helpers_match_unified_cache_surface() { + fn test_counter_and_path_helpers_match_unified_cache_surface() { let ssd_dir = tempfile::tempdir().unwrap(); let pmem_dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache( @@ -1765,7 +1798,7 @@ mod tests { } #[test] - fn parity_style_cache_traits_support_abstract_interface_consumers() { + fn style_cache_traits_support_abstract_interface_consumers() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache(CacheOptions { dram_capacity: 64, @@ -1785,7 +1818,7 @@ mod tests { assert!(cache_api.start_cache()); assert_eq!(cache_api.capacity_cache(), 64); assert_eq!( - cache_api.capacity_for_instance_cache(CacheInstanceType::kDRAM), + cache_api.capacity_for_instance_cache(CacheInstanceType::Dram), 64 ); assert_eq!(cache_api.size_cache(), 0); @@ -1797,10 +1830,10 @@ mod tests { Some(b"trait-value".to_vec()) ); assert!(cache_api.size_cache() > 0); - assert!(cache_api.used_cache(CacheInstanceType::kDRAM) > 0); - cache_api.set_capacity_for_instance_cache(CacheInstanceType::kDRAM, 8); + assert!(cache_api.used_cache(CacheInstanceType::Dram) > 0); + cache_api.set_capacity_for_instance_cache(CacheInstanceType::Dram, 8); assert_eq!( - cache_api.capacity_for_instance_cache(CacheInstanceType::kDRAM), + cache_api.capacity_for_instance_cache(CacheInstanceType::Dram), 8 ); cache_api.set_capacity_cache(4); @@ -1820,7 +1853,7 @@ mod tests { } #[test] - fn parity_style_builder_can_return_boxed_cache_interface() { + fn style_builder_can_return_boxed_cache_interface() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildCacheApi(CacheOptions { dram_capacity: 64, @@ -1837,20 +1870,20 @@ mod tests { assert_eq!(cache.capacity_cache(), 64); assert_eq!( - cache.capacity_for_instance_cache(CacheInstanceType::kDRAM), + cache.capacity_for_instance_cache(CacheInstanceType::Dram), 64 ); cache .insert_cache(key.clone(), b"boxed".to_vec(), b"boxed".len()) .unwrap(); assert_eq!(cache.lookup_cache(&key).unwrap(), Some(b"boxed".to_vec())); - assert!(cache.used_cache(CacheInstanceType::kDRAM) > 0); + assert!(cache.used_cache(CacheInstanceType::Dram) > 0); cache.reset_cache().unwrap(); assert_eq!(cache.lookup_cache(&key).unwrap(), None); } #[test] - fn parity_style_builder_can_return_boxed_zero_copy_interface() { + fn style_builder_can_return_boxed_zero_copy_interface() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCacheApi(CacheOptions { dram_capacity: 64, @@ -1876,7 +1909,7 @@ mod tests { } #[test] - fn parity_style_zero_copy_trait_preserves_pin_lifetime() { + fn style_zero_copy_trait_preserves_pin_lifetime() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache(CacheOptions { dram_capacity: 64, @@ -1907,7 +1940,7 @@ mod tests { } #[test] - fn parity_pascal_case_handle_methods_clone_and_scoped_lookup_pin_safely() { + fn pascal_case_handle_methods_clone_and_scoped_lookup_pin_safely() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildZeroCopyCache(CacheOptions { dram_capacity: 32, @@ -1982,7 +2015,7 @@ mod tests { } #[test] - fn parity_tiered_insert_uses_value_size_for_dram_admission() { + fn tiered_insert_uses_value_size_for_dram_admission() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -2014,7 +2047,7 @@ mod tests { } #[test] - fn parity_tiered_insert_pinned_uses_value_size_for_dram_handle() { + fn tiered_insert_pinned_uses_value_size_for_dram_handle() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::with_tiering_policy( dir.path(), @@ -2055,7 +2088,7 @@ mod tests { } #[test] - fn parity_cache_options_builder_constructs_equivalent_cache() { + fn cache_options_builder_constructs_equivalent_cache() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::build_zero_copy_cache(CacheOptions { dram_capacity: 64, @@ -2109,7 +2142,7 @@ mod tests { } #[test] - fn parity_cache_options_helpers_preserve_documented_policy_names() { + fn cache_options_helpers_preserve_documented_policy_names() { let dir = tempfile::tempdir().unwrap(); let options = CacheOptions::new(32, 96, 256) .with_ssd_paths(vec![dir.path().to_path_buf()]) @@ -2122,31 +2155,31 @@ mod tests { .with_ssd_instance_only(false); assert_eq!( - CacheReplacementPolicy::from_reference_name("FIFO"), + CacheReplacementPolicy::from_config_name("FIFO"), CacheReplacementPolicy::Fifo ); assert_eq!( - CacheReplacementPolicy::from_reference_name("SLRU"), + CacheReplacementPolicy::from_config_name("SLRU"), CacheReplacementPolicy::Slru ); assert_eq!( - CacheDataPlacement::from_reference_name("SideBySide"), + CacheDataPlacement::from_config_name("SideBySide"), CacheDataPlacement::SideBySide ); assert_eq!( - CacheDataPlacement::try_from_reference_name("kSideBySide").unwrap(), + CacheDataPlacement::try_from_config_name("kSideBySide").unwrap(), CacheDataPlacement::SideBySide ); assert_eq!( - CacheDataPlacement::try_from_reference_name("Tiered").unwrap(), + CacheDataPlacement::try_from_config_name("Tiered").unwrap(), CacheDataPlacement::Tiered ); assert_eq!( - DRAMPMEMDataPlacementType::try_from_reference_name("kTiered").unwrap(), - DRAMPMEMDataPlacementType::kTiered + DRAMPMEMDataPlacementType::try_from_config_name("kTiered").unwrap(), + DRAMPMEMDataPlacementType::Tiered ); assert!(matches!( - CacheDataPlacement::try_from_reference_name("bad-placement"), + CacheDataPlacement::try_from_config_name("bad-placement"), Err(CacheError::InvalidConfig(_)) )); assert_eq!(options.cache_dram_replacement_policy, "FIFO"); @@ -2176,7 +2209,7 @@ mod tests { } #[test] - fn parity_multi_tier_cache_rejects_invalid_placement_config() { + fn multi_tier_cache_rejects_invalid_placement_config() { let dir = tempfile::tempdir().unwrap(); let cache = MultiTierCache::try_new( 32, @@ -2228,7 +2261,7 @@ mod tests { } #[test] - fn parity_pascal_case_builder_factories_match_matrixcache_builder_names() { + fn pascal_case_builder_factories_match_matrixcache_builder_names() { let cache_dir = tempfile::tempdir().unwrap(); let zero_copy_dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildCache(CacheOptions { @@ -2271,7 +2304,7 @@ mod tests { } #[test] - fn parity_cache_options_zero_ssd_capacity_disables_ssd_tier() { + fn cache_options_zero_ssd_capacity_disables_ssd_tier() { let dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::build_zero_copy_cache(CacheOptions { dram_capacity: 64, @@ -2596,7 +2629,7 @@ mod tests { let cache = MultiLayerCache::with_options(options.clone()); cache .test_insert( - CacheInstanceType::kPMEM, + CacheInstanceType::Pmem, pmem_key.clone(), b"pmem-value".to_vec(), 10, @@ -2604,7 +2637,7 @@ mod tests { .unwrap(); cache .test_insert( - CacheInstanceType::kSSD, + CacheInstanceType::Ssd, ssd_key.clone(), b"ssd-value".to_vec(), 9, @@ -3081,38 +3114,38 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(64, dir.path()); - assert_eq!(CacheInstanceType::kDRAM as u8, 0); - assert_eq!(CacheInstanceType::kPMEM as u8, 1); - assert_eq!(CacheInstanceType::kSSD as u8, 2); - assert_eq!(CacheInstanceType::kUnified as u8, 3); - assert_eq!(DRAMPMEMDataPlacementType::kSideBySide as u8, 0); - assert_eq!(DRAMPMEMDataPlacementType::kTiered as u8, 1); - assert_eq!(DRAMPMEMDataPlacementType::kMaxCode as u8, 2); + assert_eq!(CacheInstanceType::Dram as u8, 0); + assert_eq!(CacheInstanceType::Pmem as u8, 1); + assert_eq!(CacheInstanceType::Ssd as u8, 2); + assert_eq!(CacheInstanceType::Unified as u8, 3); + assert_eq!(DRAMPMEMDataPlacementType::SideBySide as u8, 0); + assert_eq!(DRAMPMEMDataPlacementType::Tiered as u8, 1); + assert_eq!(DRAMPMEMDataPlacementType::MaxCode as u8, 2); assert_eq!( - DRAMPMEMDataPlacementType::FromReferenceName("kSideBySide"), - DRAMPMEMDataPlacementType::kSideBySide + DRAMPMEMDataPlacementType::FromConfigName("kSideBySide"), + DRAMPMEMDataPlacementType::SideBySide ); assert_eq!( - DRAMPMEMDataPlacementType::kTiered.AsCacheDataPlacement(), + DRAMPMEMDataPlacementType::Tiered.AsCacheDataPlacement(), CacheDataPlacement::Tiered ); assert_eq!(cache.data_placement(), CacheDataPlacement::Tiered); assert_eq!( cache.GetDRAMPMEMDataPlacementType(), - DRAMPMEMDataPlacementType::kTiered + DRAMPMEMDataPlacementType::Tiered ); cache.set_data_placement(CacheDataPlacement::SideBySide); cache.set_data_placement_threshold_bytes(32); assert_eq!(cache.data_placement(), CacheDataPlacement::SideBySide); assert_eq!( - cache.reference_data_placement_type(), - DRAMPMEMDataPlacementType::kSideBySide + cache.config_data_placement_type(), + DRAMPMEMDataPlacementType::SideBySide ); assert_eq!(cache.data_placement_threshold_bytes(), 32); - cache.SetDRAMPMEMDataPlacementType(DRAMPMEMDataPlacementType::kTiered); + cache.SetDRAMPMEMDataPlacementType(DRAMPMEMDataPlacementType::Tiered); assert_eq!(cache.GetDataPlacementType(), CacheDataPlacement::Tiered); } @@ -3257,43 +3290,43 @@ mod tests { } #[test] - fn parity_strict_replacement_policy_setter_is_pre_start_only() { + fn strict_replacement_policy_setter_is_pre_start_only() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(64, dir.path()); assert!(matches!( cache.TrySetReplacementPolicyType( - CacheInstanceType::kDRAM, + CacheInstanceType::Dram, CacheReplacementPolicy::Fifo ), Err(CacheError::AlreadyStarted) )); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::WeightedHotnessLru ); cache.Stop(); cache - .TrySetReplacementPolicyType(CacheInstanceType::kDRAM, CacheReplacementPolicy::Fifo) + .TrySetReplacementPolicyType(CacheInstanceType::Dram, CacheReplacementPolicy::Fifo) .unwrap(); cache .try_set_replacement_policy_for_tier(CacheTier::Pmem, CacheReplacementPolicy::Slru) .unwrap(); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::Fifo ); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kPMEM), + cache.GetReplacementPolicyType(CacheInstanceType::Pmem), CacheReplacementPolicy::Slru ); assert!(matches!( cache.TrySetReplacementPolicyType( - CacheInstanceType::kUnified, + CacheInstanceType::Unified, CacheReplacementPolicy::Fifo ), - Err(CacheError::UnsupportedInstance(CacheInstanceType::kUnified)) + Err(CacheError::UnsupportedInstance(CacheInstanceType::Unified)) )); assert!(cache.Start()); @@ -3351,27 +3384,27 @@ mod tests { } #[test] - fn access_record_type_matches_reference_codes_and_aliases() { - assert_eq!(CacheAccessRecordType::Put.reference_code(), 1); - assert_eq!(CacheAccessRecordType::Get.ReferenceCode(), 2); - assert_eq!(CacheAccessRecordType::Delete.reference_code(), 3); + fn access_record_type_matches_config_codes_and_aliases() { + assert_eq!(CacheAccessRecordType::Put.config_code(), 1); + assert_eq!(CacheAccessRecordType::Get.ConfigCode(), 2); + assert_eq!(CacheAccessRecordType::Delete.config_code(), 3); assert_eq!(CacheAccessRecordType::kPut, CacheAccessRecordType::Put); assert_eq!(AccessRecordType::kGet, CacheAccessRecordType::Get); - assert_eq!(AccessRecordType::kDelete.AsReferenceName(), "kDelete"); + assert_eq!(AccessRecordType::kDelete.AsConfigName(), "kDelete"); assert_eq!(AccessRecordType::kMaxCode, 4); assert_eq!( - AccessRecordType::from_reference_code(1), + AccessRecordType::from_config_code(1), Some(CacheAccessRecordType::Put) ); assert_eq!( - AccessRecordType::FromReferenceCode(2), + AccessRecordType::FromConfigCode(2), Some(CacheAccessRecordType::Get) ); assert_eq!( - AccessRecordType::from_reference_code(3), + AccessRecordType::from_config_code(3), Some(CacheAccessRecordType::Delete) ); - assert_eq!(AccessRecordType::from_reference_code(4), None); + assert_eq!(AccessRecordType::from_config_code(4), None); } #[test] @@ -3397,7 +3430,7 @@ mod tests { } #[test] - fn parity_access_record_callback_aliases_register_and_deregister() { + fn access_record_callback_aliases_register_and_deregister() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(64, dir.path()); let events = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -3496,7 +3529,7 @@ mod tests { } #[test] - fn parity_eviction_handler_aliases_disable_and_reenable_callback_delivery() { + fn eviction_handler_aliases_disable_and_reenable_callback_delivery() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(8, dir.path()); let evictions = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -3532,18 +3565,18 @@ mod tests { } #[test] - fn parity_cache_instance_dram_surface_puts_gets_peeks_deletes_and_resets() { + fn cache_instance_dram_surface_puts_gets_peeks_deletes_and_resets() { let mut instance = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); - assert_eq!(instance.StorageEngineType(), StorageEngineType::kDRAM); - assert_eq!(instance.TEST_GetStorageEngine(), StorageEngineType::kDRAM); + assert_eq!(instance.StorageEngineType(), StorageEngineType::Dram); + assert_eq!(instance.TEST_GetStorageEngine(), StorageEngineType::Dram); assert_eq!( instance.TEST_GetStorageEngineType(), - StorageEngineType::kDRAM + StorageEngineType::Dram ); instance.Start().unwrap(); @@ -3649,11 +3682,11 @@ mod tests { } #[test] - fn parity_cache_instance_put_returning_buffer_matches_put_result_surface() { + fn cache_instance_put_returning_buffer_matches_put_result_surface() { let instance = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); @@ -3671,12 +3704,12 @@ mod tests { } #[test] - fn parity_cache_instance_put_returning_buffer_reports_ssd_tier() { + fn cache_instance_put_returning_buffer_reports_ssd_tier() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![dir.path().to_path_buf()], ); @@ -3693,12 +3726,12 @@ mod tests { } #[test] - fn parity_cache_instance_pmem_surface_uses_exact_pmem_tier() { + fn cache_instance_pmem_surface_uses_exact_pmem_tier() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 32, - ReplacementPolicyType::kSLRU, - StorageEngineType::kPMEM, + ReplacementPolicyType::Slru, + StorageEngineType::Pmem, vec![dir.path().to_path_buf()], ); @@ -3717,18 +3750,18 @@ mod tests { } #[test] - fn parity_l1_cache_implement_pulls_dram_then_pmem_without_replacement_access() { + fn l1_cache_implement_pulls_dram_then_pmem_without_replacement_access() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); let pmem_dir = tempfile::tempdir().unwrap(); let pmem = CacheInstance::new( 64, - ReplacementPolicyType::kSLRU, - StorageEngineType::kPMEM, + ReplacementPolicyType::Slru, + StorageEngineType::Pmem, vec![pmem_dir.path().to_path_buf()], ); dram.Put("shared", b"dram-value".to_vec()).unwrap(); @@ -3756,11 +3789,11 @@ mod tests { } #[test] - fn parity_l1_cache_implement_allows_absent_pmem_instance() { + fn l1_cache_implement_allows_absent_pmem_instance() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); dram.Put("dram-only", b"value".to_vec()).unwrap(); @@ -3780,18 +3813,18 @@ mod tests { } #[test] - fn parity_l2_cache_policy_access_tail_and_write_match_arc_flow() { + fn l2_cache_policy_access_tail_and_write_match_arc_flow() { let dram = CacheInstance::new( 128, - ReplacementPolicyType::kWeightedHotnessLru, - StorageEngineType::kDRAM, + ReplacementPolicyType::WeightedHotnessLru, + StorageEngineType::Dram, vec![], ); let l2_dir = tempfile::tempdir().unwrap(); let l2 = CacheInstance::new( 256, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.path().to_path_buf()], ); dram.Put("cold-a", b"alpha".to_vec()).unwrap(); @@ -3841,18 +3874,18 @@ mod tests { } #[test] - fn parity_l2_cache_policy_eviction_queue_duplicate_and_overflow_paths() { + fn l2_cache_policy_eviction_queue_duplicate_and_overflow_paths() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, vec![], ); let l2_dir = tempfile::tempdir().unwrap(); let l2 = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.path().to_path_buf()], ); let l1 = L1CacheImplement::new(dram, None); @@ -3916,14 +3949,14 @@ mod tests { ) -> L2CachePolicy { let dram = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, vec![], ); let l2 = CacheInstance::new( 256, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.to_path_buf()], ); let l1 = L1CacheImplement::new(dram, None); @@ -3933,7 +3966,7 @@ mod tests { } #[test] - fn parity_l2_cache_policy_access_buffering_modes_and_drop_on_full() { + fn l2_cache_policy_access_buffering_modes_and_drop_on_full() { let dir = tempfile::tempdir().unwrap(); let mut policy = l2_test_policy(dir.path(), 8, 2, 8, 8); policy.Start(); @@ -3962,7 +3995,7 @@ mod tests { } #[test] - fn parity_l2_cache_policy_eviction_handler_is_off_by_default() { + fn l2_cache_policy_eviction_handler_is_off_by_default() { let dir = tempfile::tempdir().unwrap(); let mut policy = l2_test_policy(dir.path(), 8, 8, 8, 8); policy.Start(); @@ -3984,7 +4017,7 @@ mod tests { } #[test] - fn parity_l2_cache_policy_poll_paces_passes_by_interval() { + fn l2_cache_policy_poll_paces_passes_by_interval() { let dir = tempfile::tempdir().unwrap(); let mut policy = l2_test_policy(dir.path(), 8, 8, 8, 8); policy.Start(); @@ -4019,18 +4052,18 @@ mod tests { } #[test] - fn parity_l2_cache_policy_factory_uses_reference_default_sizing() { + fn l2_cache_policy_factory_uses_config_default_sizing() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, vec![], ); let l2_dir = tempfile::tempdir().unwrap(); let l2 = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.path().to_path_buf()], ); let l1 = L1CacheImplement::new(dram, None); @@ -4048,18 +4081,18 @@ mod tests { } #[test] - fn parity_l2_cache_policy_factory_builds_started_policy_surface() { + fn l2_cache_policy_factory_builds_started_policy_surface() { let dram = CacheInstance::new( 64, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, vec![], ); let l2_dir = tempfile::tempdir().unwrap(); let l2 = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![l2_dir.path().to_path_buf()], ); let l1 = L1CacheImplement::new(dram, None); @@ -4072,17 +4105,17 @@ mod tests { } #[test] - fn parity_cache_instance_ssd_surface_recovers_persistent_index() { + fn cache_instance_ssd_surface_recovers_persistent_index() { let dir = tempfile::tempdir().unwrap(); let paths = vec![dir.path().to_path_buf()]; let instance = CacheInstance::new( 128, - ReplacementPolicyType::kWeightedHotnessLru, - StorageEngineType::kSSD, + ReplacementPolicyType::WeightedHotnessLru, + StorageEngineType::Ssd, paths.clone(), ); - assert_eq!(instance.StorageEngineType(), StorageEngineType::kSSD); - assert_eq!(instance.TEST_GetStorageEngine(), StorageEngineType::kSSD); + assert_eq!(instance.StorageEngineType(), StorageEngineType::Ssd); + assert_eq!(instance.TEST_GetStorageEngine(), StorageEngineType::Ssd); instance.Put("ssd-key", b"ssd-value".to_vec()).unwrap(); assert_eq!( instance.Get("ssd-key").unwrap(), @@ -4097,8 +4130,8 @@ mod tests { let restarted = CacheInstance::new( 128, - ReplacementPolicyType::kWeightedHotnessLru, - StorageEngineType::kSSD, + ReplacementPolicyType::WeightedHotnessLru, + StorageEngineType::Ssd, paths, ); let report = restarted.RecoverData().unwrap(); @@ -4110,12 +4143,12 @@ mod tests { } #[test] - fn parity_cache_instance_ssd_put_bypass_storage_writes_value() { + fn cache_instance_ssd_put_bypass_storage_writes_value() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![dir.path().to_path_buf()], ); @@ -4141,12 +4174,12 @@ mod tests { } #[test] - fn parity_cache_instance_ssd_update_rewrites_guarded_block() { + fn cache_instance_ssd_update_rewrites_guarded_block() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![dir.path().to_path_buf()], ); @@ -4190,11 +4223,11 @@ mod tests { } #[test] - fn parity_cache_instance_eviction_and_metric_handlers_follow_status() { + fn cache_instance_eviction_and_metric_handlers_follow_status() { let instance = CacheInstance::new( 8, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); let evictions = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -4230,7 +4263,7 @@ mod tests { } #[test] - fn parity_cache_buffer_exposes_key_data_size_and_set_key() { + fn cache_buffer_exposes_key_data_size_and_set_key() { let mut buffer = StringBuffer::string("hello"); assert_eq!(buffer.Key(), ""); buffer.SetKey("buffer-key"); @@ -4249,7 +4282,7 @@ mod tests { } #[test] - fn parity_iobuf_buffer_owns_data_and_converts_to_cache_buffer() { + fn iobuf_buffer_owns_data_and_converts_to_cache_buffer() { let mut buffer = IOBufBuffer::new(b"iobuf-value".to_vec()); assert_eq!(buffer.Key(), ""); buffer.SetKey("iobuf-key"); @@ -4265,8 +4298,8 @@ mod tests { let instance = CacheInstance::new( 128, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); let inserted = instance.PutBuffer(converted).unwrap(); @@ -4278,9 +4311,9 @@ mod tests { } #[test] - fn parity_raw_buffer_exposes_owned_data_reset_and_cache_buffer_conversion() { + fn raw_buffer_exposes_owned_data_reset_and_cache_buffer_conversion() { let mut raw = - RawBuffer::with_storage_engine(b"raw-value".to_vec(), StorageEngineType::kPMEM, true); + RawBuffer::with_storage_engine(b"raw-value".to_vec(), StorageEngineType::Pmem, true); assert_eq!(raw.Key(), ""); raw.SetKey("raw-key"); assert_eq!(raw.Key(), "raw-key"); @@ -4288,7 +4321,7 @@ mod tests { assert_eq!(raw.DataPtr(), raw.Data().as_ptr()); assert_eq!(raw.Value(), b"raw-value"); assert_eq!(raw.Size(), 9); - assert_eq!(raw.storage_engine(), Some(StorageEngineType::kPMEM)); + assert_eq!(raw.storage_engine(), Some(StorageEngineType::Pmem)); assert!(raw.async_delete()); let converted: CacheBuffer = raw.into(); @@ -4308,7 +4341,7 @@ mod tests { } #[test] - fn parity_string_view_buffer_tracks_key_and_size_without_holding_data() { + fn string_view_buffer_tracks_key_and_size_without_holding_data() { let mut view = StringViewBuffer::new(4096); assert_eq!(view.Key(), ""); view.SetKey("ssd-view"); @@ -4324,11 +4357,11 @@ mod tests { } #[test] - fn parity_cache_instance_accepts_raw_buffer_conversion_for_put_buffer() { + fn cache_instance_accepts_raw_buffer_conversion_for_put_buffer() { let instance = CacheInstance::new( 32, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); let mut raw = RawBuffer::new(b"raw-put".to_vec()); @@ -4344,11 +4377,11 @@ mod tests { } #[test] - fn parity_cache_instance_buffer_put_get_and_update_are_guarded() { + fn cache_instance_buffer_put_get_and_update_are_guarded() { let mut instance = CacheInstance::new( 32, - ReplacementPolicyType::kFIFO, - StorageEngineType::kDRAM, + ReplacementPolicyType::Fifo, + StorageEngineType::Dram, Vec::new(), ); @@ -4427,7 +4460,7 @@ mod tests { } #[test] - fn parity_flexible_cache_wraps_configurable_cache_instance_for_strings() { + fn flexible_cache_wraps_configurable_cache_instance_for_strings() { let cache = MatrixCacheBuilder::BuildFlexibleCache( 8, "fifo", @@ -4436,8 +4469,8 @@ mod tests { Vec::::new(), ); - assert_eq!(cache.policy(), ReplacementPolicyType::kFIFO); - assert_eq!(cache.engine(), StorageEngineType::kDRAM); + assert_eq!(cache.policy(), ReplacementPolicyType::Fifo); + assert_eq!(cache.engine(), StorageEngineType::Dram); assert!(cache.Start()); cache.Insert("first", "12345678".to_string(), 8).unwrap(); assert_eq!(cache.Lookup("first").unwrap(), Some("12345678".to_string())); @@ -4456,7 +4489,7 @@ mod tests { } #[test] - fn parity_blockcache_facade_enforces_lifecycle_and_clears_ssd_paths() { + fn blockcache_facade_enforces_lifecycle_and_clears_ssd_paths() { let dir = tempfile::tempdir().unwrap(); let ssd_path = dir.path().join("blockcache-ssd"); fs::create_dir_all(&ssd_path).unwrap(); @@ -4492,7 +4525,7 @@ mod tests { } #[test] - fn parity_flexible_cache_uses_selected_ssd_paths_and_recovers() { + fn flexible_cache_uses_selected_ssd_paths_and_recovers() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().to_string_lossy().to_string(); let cache = MatrixCacheBuilder::BuildFlexibleCacheFromPathStrings( @@ -4503,8 +4536,8 @@ mod tests { vec![path.clone()], ); - assert_eq!(cache.policy(), ReplacementPolicyType::kWeightedHotnessLru); - assert_eq!(cache.engine(), StorageEngineType::kSSD); + assert_eq!(cache.policy(), ReplacementPolicyType::WeightedHotnessLru); + assert_eq!(cache.engine(), StorageEngineType::Ssd); assert_eq!(cache.paths(), &[PathBuf::from(&path)]); cache.Insert("ssd-key", "ssd-value".to_string(), 9).unwrap(); assert_eq!( @@ -4529,7 +4562,7 @@ mod tests { } #[test] - fn parity_pmem_cache_instance_persists_and_recovers_from_configured_path() { + fn pmem_cache_instance_persists_and_recovers_from_configured_path() { let dir = tempfile::tempdir().unwrap(); let pmem_path = dir.path().join("pmem-device"); let paths = vec![pmem_path.to_string_lossy().to_string()]; @@ -4540,7 +4573,7 @@ mod tests { paths.clone(), Vec::::new(), ); - assert_eq!(cache.engine(), StorageEngineType::kPMEM); + assert_eq!(cache.engine(), StorageEngineType::Pmem); cache.Insert("pmem-a", "value-a".to_string(), 7).unwrap(); cache.Insert("pmem-b", "value-b".to_string(), 7).unwrap(); assert_eq!(cache.Lookup("pmem-a").unwrap(), Some("value-a".to_string())); @@ -4591,14 +4624,14 @@ mod tests { .with_auto_recover_on_start(true); let cache = MultiLayerCache::try_with_options(options.clone()).unwrap(); cache - .test_insert(CacheInstanceType::kPMEM, key.clone(), b"pmem".to_vec(), 4) + .test_insert(CacheInstanceType::Pmem, key.clone(), b"pmem".to_vec(), 4) .unwrap(); let restarted = MultiLayerCache::try_with_options(options).unwrap(); assert_eq!(restarted.peek_tier(&key), Some(CacheReadTier::Pmem)); assert_eq!( restarted - .test_acquire(CacheInstanceType::kPMEM, &key) + .test_acquire(CacheInstanceType::Pmem, &key) .unwrap() .unwrap() .value(), @@ -4616,11 +4649,11 @@ mod tests { let key = CacheKey::string(7, "general-remove-pmem"); cache - .test_insert(CacheInstanceType::kPMEM, key.clone(), b"pmem".to_vec(), 4) + .test_insert(CacheInstanceType::Pmem, key.clone(), b"pmem".to_vec(), 4) .unwrap(); assert_eq!( cache - .test_acquire(CacheInstanceType::kPMEM, &key) + .test_acquire(CacheInstanceType::Pmem, &key) .unwrap() .unwrap() .value(), @@ -4633,13 +4666,13 @@ mod tests { let report = restarted.recover_pmem_index().unwrap(); assert_eq!(report.recovered_files, 0); assert!(restarted - .test_acquire(CacheInstanceType::kPMEM, &key) + .test_acquire(CacheInstanceType::Pmem, &key) .unwrap() .is_none()); } #[test] - fn parity_flexible_cache_multi_ssd_uses_all_paths_and_recovers() { + fn flexible_cache_multi_ssd_uses_all_paths_and_recovers() { let dir_a = tempfile::tempdir().unwrap(); let dir_b = tempfile::tempdir().unwrap(); let path_a = dir_a.path().to_string_lossy().to_string(); @@ -4653,7 +4686,7 @@ mod tests { paths.clone(), ); - assert_eq!(cache.engine(), StorageEngineType::kMultiSSD); + assert_eq!(cache.engine(), StorageEngineType::MultiSsd); assert_eq!( cache.paths(), &[PathBuf::from(&path_a), PathBuf::from(&path_b)] @@ -4720,7 +4753,7 @@ mod tests { } #[test] - fn parity_multi_tier_cache_wrapper_builds_unified_cache_from_constructor_knobs() { + fn multi_tier_cache_wrapper_builds_unified_cache_from_constructor_knobs() { let pmem_dir = tempfile::tempdir().unwrap(); let ssd_dir = tempfile::tempdir().unwrap(); let cache = MatrixCacheBuilder::BuildMultiTierCacheFromPathStrings( @@ -4736,8 +4769,8 @@ mod tests { "rocksdb", ); - assert_eq!(cache.policy(), ReplacementPolicyType::kFIFO); - assert_eq!(cache.ssd_storage_engine(), StorageEngineType::kSSD); + assert_eq!(cache.policy(), ReplacementPolicyType::Fifo); + assert_eq!(cache.ssd_storage_engine(), StorageEngineType::Ssd); assert!(!cache.eviction_enabled()); assert_eq!(cache.options().dram_capacity, 16); assert_eq!(cache.options().pmem_capacity, 32); @@ -4747,9 +4780,9 @@ mod tests { "SideBySide" ); assert_eq!(cache.options().cache_dram_pmem_data_placement_threshold, 8); - assert_eq!(cache.inner().GetCapacity(CacheInstanceType::kDRAM), 16); - assert_eq!(cache.inner().GetCapacity(CacheInstanceType::kPMEM), 32); - assert_eq!(cache.inner().GetCapacity(CacheInstanceType::kSSD), 128); + assert_eq!(cache.inner().GetCapacity(CacheInstanceType::Dram), 16); + assert_eq!(cache.inner().GetCapacity(CacheInstanceType::Pmem), 32); + assert_eq!(cache.inner().GetCapacity(CacheInstanceType::Ssd), 128); assert!(!cache.inner().EvictionHandlerEnabled()); assert!(cache.Start()); @@ -5459,6 +5492,111 @@ mod tests { assert_eq!(remaining.iter().filter(|value| value.is_some()).count(), 2); } + /// Every tier's key order has to hold exactly the keys that tier holds. + /// + /// Victim selection reads the order rather than the tier map, so a key the + /// order has lost can never be evicted and a key it lists but the tier no + /// longer holds wastes a round of selection. Neither shows up as a wrong + /// answer until the cache is under pressure, so this checks the invariant + /// directly after a workload that exercises every path that edits it. + #[test] + fn tier_orders_hold_exactly_the_keys_their_tiers_hold() { + let dir = tempfile::tempdir().unwrap(); + let cache = MultiLayerCache::with_options(CacheOptions { + dram_capacity: 512, + pmem_capacity: 512, + ssd_capacity: 4096, + ssd_paths: vec![dir.path().to_path_buf()], + ..CacheOptions::default() + }); + cache.start().unwrap(); + + let keys = (0..64) + .map(|index| CacheKey::string(0, &format!("order-invariant-{index:04}"))) + .collect::>(); + + // Fill past capacity so entries evict and demote between tiers. + for key in &keys { + cache.put(key.clone(), vec![b'z'; 16]).unwrap(); + } + // Reads promote and refill, which rewrites tier membership. + for key in keys.iter().step_by(3) { + let _ = cache.get(key).unwrap(); + } + // Explicit removals and an invalidation take their own paths. + for key in keys.iter().step_by(7) { + cache.remove(key).unwrap(); + } + cache.invalidate(&keys[1]).unwrap(); + cache.invalidate_memory_only(&keys[2]); + // A shrink evicts a batch in one pass. + cache.set_capacity_for_tier(CacheTier::Memory, 128); + + let inner = cache.inner.read().expect("cache lock poisoned"); + let order_keys = |order: &CacheKeyOrder| { + order.iter().cloned().collect::>() + }; + assert_eq!( + order_keys(&inner.memory_order), + inner.memory.keys().cloned().collect::>(), + "memory order and memory tier disagree" + ); + assert_eq!( + order_keys(&inner.pmem_order), + inner.pmem.keys().cloned().collect::>(), + "pmem order and pmem tier disagree" + ); + assert_eq!( + order_keys(&inner.disk_order), + inner.disk_index.keys().cloned().collect::>(), + "disk order and disk tier disagree" + ); + } + + /// First-in first-out eviction has to keep going until the tier is back + /// under its budget, however many entries that takes. + /// + /// Selection reads the tier's key order, so a victim that is taken but + /// left in that order gets picked again on the next round. The second pick + /// frees nothing, the loop reads that as no progress and stops, and the + /// tier is left over capacity with one entry removed instead of three. + #[test] + fn fifo_capacity_shrink_evicts_the_whole_overage_in_one_pass() { + let dir = tempfile::tempdir().unwrap(); + // Memory only: a victim with a tier below it is demoted rather than + // dropped, and would still read back. + let cache = MultiLayerCache::with_options(CacheOptions { + dram_capacity: 40, + pmem_capacity: 0, + ssd_capacity: 0, + ssd_paths: vec![dir.path().to_path_buf()], + cache_dram_replacement_policy: "FIFO".to_string(), + ..CacheOptions::default() + }); + cache.start().unwrap(); + + let keys = (0..5) + .map(|index| CacheKey::string(0, &format!("fifo-shrink-{index}"))) + .collect::>(); + for key in &keys { + cache.put(key.clone(), vec![b'x'; 8]).unwrap(); + } + assert_eq!(cache.size_for_tier(CacheTier::Memory), 40); + + // Room for two of the five entries, so three have to go at once. + cache.set_capacity_for_tier(CacheTier::Memory, 16); + + assert!( + cache.size_for_tier(CacheTier::Memory) <= 16, + "tier left over capacity at {} bytes", + cache.size_for_tier(CacheTier::Memory) + ); + let remaining = cache.get_batch(&keys).unwrap(); + assert_eq!(remaining.iter().filter(|value| value.is_some()).count(), 2); + // First in, first out: the two survivors are the ones written last. + assert!(remaining[3].is_some() && remaining[4].is_some()); + } + // shared-corpus: storage_cache_refill; #[test] fn weighted_ssd_eviction_preserves_hot_entries() { @@ -5827,7 +5965,7 @@ mod tests { } #[test] - fn pinned_handle_clone_with_cache_matches_reference_explicit_clone_semantics() { + fn pinned_handle_clone_with_cache_matches_config_explicit_clone_semantics() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(8, dir.path()); let key = CacheKey::string(1, "clone"); @@ -5918,7 +6056,7 @@ mod tests { } #[test] - fn scoped_lookup_matches_reference_found_and_auto_release_semantics() { + fn scoped_lookup_matches_config_found_and_auto_release_semantics() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(16, dir.path()); let key = CacheKey::string(1, "scoped-lookup"); @@ -6015,7 +6153,7 @@ mod tests { } #[test] - fn parity_remove_keeps_removed_pinned_entry_counted_until_release() { + fn remove_keeps_removed_pinned_entry_counted_until_release() { let dir = tempfile::tempdir().unwrap(); let cache = MultiLayerCache::new(16, dir.path()); let key = CacheKey::page_with_slot(1, 10, 0, 4, Some(7)); @@ -6362,7 +6500,7 @@ mod tests { } #[test] - fn parity_base_lru_list_tracks_mru_and_tail_eviction() { + fn base_lru_list_tracks_mru_and_tail_eviction() { let mut list = BaseLRUList::new(2); list.Put("a".to_string()); list.Put("b".to_string()); @@ -6379,7 +6517,7 @@ mod tests { } #[test] - fn parity_ghost_lru_list_downgrades_data_to_ghost_tail() { + fn ghost_lru_list_downgrades_data_to_ghost_tail() { let mut list = GhostLRUList::new(1); list.Put("hot".to_string()); list.Put("cold".to_string()); @@ -6396,7 +6534,7 @@ mod tests { } #[test] - fn parity_arc_list_promotes_hits_and_keeps_bounded_data_size() { + fn arc_list_promotes_hits_and_keeps_bounded_data_size() { let mut arc = ArcList::new(2); arc.Put("a".to_string()); arc.Put("b".to_string()); @@ -6418,7 +6556,7 @@ mod tests { } #[test] - fn parity_arc_list_hit_on_fetch_data_promotes_to_active() { + fn arc_list_hit_on_fetch_data_promotes_to_active() { let mut arc = ArcList::new(4); arc.Put("a".to_string()); assert_eq!(arc.GetFetchDataTail(8), vec!["a".to_string()]); @@ -6432,7 +6570,7 @@ mod tests { } #[test] - fn parity_arc_list_ghost_hits_adapt_the_fetch_active_split() { + fn arc_list_ghost_hits_adapt_the_fetch_active_split() { // Capacity 2 starts split evenly, one slot each side. let mut arc = ArcList::new(2); assert_eq!(arc.FetchCapacity(), 1); @@ -6463,7 +6601,7 @@ mod tests { } #[test] - fn parity_arc_list_drops_fetch_tail_outright_when_its_ghost_is_empty() { + fn arc_list_drops_fetch_tail_outright_when_its_ghost_is_empty() { let mut arc = ArcList::new(2); arc.Put("a".to_string()); arc.Put("b".to_string()); @@ -6503,7 +6641,7 @@ mod tests { } #[test] - fn parity_replacement_arc_exposes_active_and_fetch_tail_surface() { + fn replacement_arc_exposes_active_and_fetch_tail_surface() { let mut policy = ReplacementArc::new(2); assert!(!policy.is_initialized()); policy.Init().unwrap(); @@ -6528,55 +6666,55 @@ mod tests { } #[test] - fn parity_storage_engine_type_preserves_codes_and_aliases() { + fn storage_engine_type_preserves_codes_and_aliases() { assert_eq!( - StorageEngineType::from_reference_name("kDRAMStorageEngine"), - StorageEngineType::kDRAM + StorageEngineType::from_config_name("kDRAMStorageEngine"), + StorageEngineType::Dram ); assert_eq!( - StorageEngineType::from_reference_name("kSimpleStorageEngine"), - StorageEngineType::kSimple + StorageEngineType::from_config_name("kSimpleStorageEngine"), + StorageEngineType::Simple ); - assert_eq!(StorageEngineType::kMultiSSD.ReferenceCode(), 4); + assert_eq!(StorageEngineType::MultiSsd.ConfigCode(), 4); assert_eq!( - StorageEngineType::kSimple.AsReferenceEnumName(), + StorageEngineType::Simple.AsConfigEnumName(), "kSimpleStorageEngine" ); assert_eq!( - StorageEngineType::from_reference_name("rocksdb"), - StorageEngineType::kSSD + StorageEngineType::from_config_name("rocksdb"), + StorageEngineType::Ssd ); assert_eq!( - StorageEngineType::from_reference_name("kSSDRocksDBStorageEngine"), - StorageEngineType::kSSD + StorageEngineType::from_config_name("kSSDRocksDBStorageEngine"), + StorageEngineType::Ssd ); - assert_eq!(SSDEngineType::kRocksDB as u8, 0); + assert_eq!(SSDEngineType::RocksDb as u8, 0); assert_eq!( - SSDEngineType::FromReferenceName("rocksdb"), - SSDEngineType::kRocksDB + SSDEngineType::FromConfigName("rocksdb"), + SSDEngineType::RocksDb ); - assert_eq!(SSDEngineType::kRocksDB.AsReferenceName(), "RocksDB"); - assert_eq!(WriteBufferType::kUserDataBuf as u8, 0); - assert_eq!(WriteBufferType::kMetaDataBuf as u8, 1); - assert_eq!(WriteBufferType::kGCBuf as u8, 2); - assert_eq!(WriteBufferType::kCodecDataBuf as u8, 3); - assert_eq!(DataType::DATA as u8, 1); - assert_eq!(DataType::META_LOG as u8, 2); - assert_eq!(GCMode::LOSSY as u8, 1); - assert_eq!(GCMode::LOSSLESS as u8, 10); - assert_eq!(RecordStateType::kSoftDel as u8, 0x0); - assert_eq!(RecordStateType::kNormal as u8, 0x1); - assert_eq!(RecordStateType::kPinned as u8, 0x2); - assert_eq!(RecordStateType::kMaxCode as u8, 0xf); + assert_eq!(SSDEngineType::RocksDb.AsConfigName(), "RocksDB"); + assert_eq!(WriteBufferType::UserDataBuf as u8, 0); + assert_eq!(WriteBufferType::MetaDataBuf as u8, 1); + assert_eq!(WriteBufferType::GcBuf as u8, 2); + assert_eq!(WriteBufferType::CodecDataBuf as u8, 3); + assert_eq!(DataType::Data as u8, 1); + assert_eq!(DataType::MetaLog as u8, 2); + assert_eq!(GCMode::Lossy as u8, 1); + assert_eq!(GCMode::Lossless as u8, 10); + assert_eq!(RecordStateType::SoftDel as u8, 0x0); + assert_eq!(RecordStateType::Normal as u8, 0x1); + assert_eq!(RecordStateType::Pinned as u8, 0x2); + assert_eq!(RecordStateType::MaxCode as u8, 0xf); } #[test] - fn parity_write_buffer_and_encoder_preserve_layout_size_semantics() { - let mut buffer = WriteBuffer::new(WriteBufferType::kUserDataBuf, 128); + fn write_buffer_and_encoder_preserve_layout_size_semantics() { + let mut buffer = WriteBuffer::new(WriteBufferType::UserDataBuf, 128); buffer.PushBack("a", b"one".to_vec()); buffer.PushBack("bb", b"twotwo".to_vec()); assert_eq!(buffer.Capacity(), 128); - assert_eq!(buffer.BufType(), WriteBufferType::kUserDataBuf); + assert_eq!(buffer.BufType(), WriteBufferType::UserDataBuf); assert_eq!(buffer.Count(), 2); assert_eq!(buffer.KeySize(), 3); assert_eq!(buffer.ValueSize(), 9); @@ -6609,7 +6747,7 @@ mod tests { } #[test] - fn parity_mem_storage_crc_is_castagnoli_and_covers_the_length_header() { + fn mem_storage_crc_is_castagnoli_and_covers_the_length_header() { // Pin the algorithm to its published check value rather than to // whatever this implementation happens to emit. assert_eq!(crc32c(b"123456789"), 0xe306_9283); @@ -6643,13 +6781,13 @@ mod tests { } #[test] - fn parity_mem_storage_layout_round_trips_key_value_and_crc() { + fn mem_storage_layout_round_trips_key_value_and_crc() { let crc = MemStorage::ComputeCRC("layout-key", b"layout-value"); let record = MemStorage::DoPutWithCRC("layout-key", b"layout-value", crc).unwrap(); assert_eq!(MemStorage::GetKeyFromData(&record).unwrap(), "layout-key"); let buffer = - MemStorage::CreateCacheBufferFromData(&record, StorageEngineType::kSimple, false) + MemStorage::CreateCacheBufferFromData(&record, StorageEngineType::Simple, false) .unwrap(); assert_eq!(buffer.Key(), "layout-key"); assert_eq!(buffer.Data(), b"layout-value"); @@ -6657,7 +6795,7 @@ mod tests { } #[test] - fn parity_mem_storage_allocator_handle_models_payload_pointer_and_delete() { + fn mem_storage_allocator_handle_models_payload_pointer_and_delete() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(256); let handle = MemStorage::DoPutToAllocator(&mut allocator, "alloc-key", b"alloc-value").unwrap(); @@ -6685,7 +6823,7 @@ mod tests { let buffer = MemStorage::CreateCacheBufferFromAllocatorData( &allocator, handle, - StorageEngineType::kSimple, + StorageEngineType::Simple, false, ) .unwrap(); @@ -6701,7 +6839,7 @@ mod tests { } #[test] - fn parity_mem_storage_allocator_path_rejects_corrupt_crc_before_write() { + fn mem_storage_allocator_path_rejects_corrupt_crc_before_write() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(256); let crc = MemStorage::ComputeCRC("alloc-key", b"alloc-value"); @@ -6716,7 +6854,7 @@ mod tests { } #[test] - fn parity_mem_storage_allocator_surface_works_for_je_and_pool_allocators() { + fn mem_storage_allocator_surface_works_for_je_and_pool_allocators() { let mut je = JeAllocator::with_capacity(256); let je_handle = MemStorage::DoPutToAllocator(&mut je, "je-key", b"je-value").unwrap(); assert_eq!(je_handle.PayloadOffset(), MemStorage::HEADER_BYTES); @@ -6738,7 +6876,7 @@ mod tests { } #[test] - fn parity_simple_storage_engine_lifecycle_put_peek_delete_and_recover() { + fn simple_storage_engine_lifecycle_put_peek_delete_and_recover() { #[derive(Default)] struct Collector { recovered: Vec<(String, Vec)>, @@ -6752,7 +6890,7 @@ mod tests { let mut engine = StorageEngineSimple::with_capacity(1024); assert_eq!(engine.Capacity(), 1024); - assert_eq!(engine.StorageEngineType(), StorageEngineType::kSimple); + assert_eq!(engine.StorageEngineType(), StorageEngineType::Simple); engine.SetCapacity(2048); assert_eq!(engine.Capacity(), 2048); assert!(!engine.is_started()); @@ -6817,7 +6955,7 @@ mod tests { let mut engine = StorageEngineRocksDB::new(&db_path_str); assert_eq!(engine.Path(), db_path_str); - assert_eq!(engine.StorageEngineType(), StorageEngineType::kSSD); + assert_eq!(engine.StorageEngineType(), StorageEngineType::Ssd); assert_eq!( engine.SsdBackendName(), if cfg!(feature = "rocksdb-ssd") { @@ -6912,7 +7050,7 @@ mod tests { } #[test] - fn parity_storage_recover_callback_mock_tracks_last_key_and_count() { + fn storage_recover_callback_mock_tracks_last_key_and_count() { let mut engine = StorageEngineSimple::with_capacity(1024); assert!(engine.Start()); engine.Put("first", b"111".to_vec()).unwrap(); @@ -6945,7 +7083,7 @@ mod tests { } #[test] - fn parity_gc_copy_callback_mock_replaces_buffers_with_guarded_old_data() { + fn gc_copy_callback_mock_replaces_buffers_with_guarded_old_data() { let mut callback = GCCopyCallbackMock::new(); let mut old = CacheBuffer::new(b"old-value".to_vec()); old.SetKey("alpha"); @@ -6984,7 +7122,7 @@ mod tests { } #[test] - fn parity_log_allocator_gc_listener_mock_updates_maps_and_frees_old_ptr() { + fn log_allocator_gc_listener_mock_updates_maps_and_frees_old_ptr() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); let old_ptr = allocator.Allocate(8).unwrap(); let new_ptr = allocator.Allocate(8).unwrap(); @@ -7016,7 +7154,7 @@ mod tests { } #[test] - fn parity_pmem_recover_listener_dedupes_records_before_callback() { + fn pmem_recover_listener_dedupes_records_before_callback() { #[derive(Default)] struct Collector { recovered: Vec<(String, Vec)>, @@ -7049,7 +7187,7 @@ mod tests { } #[test] - fn parity_pmem_storage_test_hooks_put_to_numa_and_report_recover_stats() { + fn pmem_storage_test_hooks_put_to_numa_and_report_recover_stats() { let mut engine = StorageEnginePMem::with_capacity(1024); assert!(engine.Start()); engine.TEST_JoinPmemWriteExecutor(); @@ -7068,12 +7206,12 @@ mod tests { } #[test] - fn parity_ssd_fifo_keeps_insertion_order_when_a_key_is_rewritten() { + fn ssd_fifo_keeps_insertion_order_when_a_key_is_rewritten() { let dir = tempfile::tempdir().unwrap(); let instance = CacheInstance::new( 520, - ReplacementPolicyType::kFIFO, - StorageEngineType::kSSD, + ReplacementPolicyType::Fifo, + StorageEngineType::Ssd, vec![dir.path().to_path_buf()], ); instance.Start().unwrap(); @@ -7102,7 +7240,7 @@ mod tests { } #[test] - fn parity_multi_ssd_selects_the_device_with_the_shared_hash() { + fn multi_ssd_selects_the_device_with_the_shared_hash() { let dir = tempfile::tempdir().unwrap(); let dev = |name: &str| dir.path().join(name).to_string_lossy().to_string(); let paths = vec![dev("ssd-a"), dev("ssd-b"), dev("ssd-c")]; @@ -7134,7 +7272,7 @@ mod tests { } #[test] - fn parity_multi_ssd_requires_devices_and_hashes_keys_to_storage() { + fn multi_ssd_requires_devices_and_hashes_keys_to_storage() { let mut empty = StorageEngineMultiSSD::new(Vec::::new(), 1024); assert!(!empty.Start()); assert!(matches!( @@ -7170,7 +7308,7 @@ mod tests { } #[test] - fn parity_multi_ssd_recovers_resets_and_manages_devices() { + fn multi_ssd_recovers_resets_and_manages_devices() { struct Collector { recovered: Vec<(String, Vec)>, } @@ -7184,7 +7322,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let dev = |name: &str| dir.path().join(name).to_string_lossy().to_string(); let mut engine = StorageEngineMultiSSD::new(vec![dev("ssd-a"), dev("ssd-b")], 2048); - assert_eq!(engine.StorageEngineType(), StorageEngineType::kMultiSSD); + assert_eq!(engine.StorageEngineType(), StorageEngineType::MultiSsd); assert!(engine.Start()); engine.Put("first", b"111".to_vec()).unwrap(); engine.Put("second", b"222".to_vec()).unwrap(); @@ -7327,26 +7465,26 @@ mod tests { #[test] fn storage_config_storage_engine_type_conversions() { use StorageEngineType::*; - for ty in [kDRAM, kPMEM, kSSD, kSimple, kMultiSSD] { + for ty in [Dram, Pmem, Ssd, Simple, MultiSsd] { // code conversion round-trips, and every variant has a display name - assert_eq!(StorageEngineType::from_reference_code(ty.reference_code()), ty); - assert!(!ty.as_reference_name().is_empty()); + assert_eq!(StorageEngineType::from_config_code(ty.config_code()), ty); + assert!(!ty.as_config_name().is_empty()); } // recognized name spellings parse to the expected engine - assert_eq!(StorageEngineType::from_reference_name("ssd"), kSSD); - assert_eq!(StorageEngineType::from_reference_name("kRocksDB"), kSSD); - assert_eq!(StorageEngineType::from_reference_name("pmem"), kPMEM); - assert_eq!(StorageEngineType::from_reference_name("multi_ssd"), kMultiSSD); - assert_eq!(StorageEngineType::from_reference_name("simple"), kSimple); - assert_eq!(StorageEngineType::from_reference_name("dram"), kDRAM); - assert!(kSSD.is_ssd_like()); - assert!(kMultiSSD.is_ssd_like()); - assert!(!kDRAM.is_ssd_like()); - assert_eq!(kPMEM.canonical_instance_type(), CacheInstanceType::kPMEM); - assert_eq!(kDRAM.canonical_instance_type(), CacheInstanceType::kDRAM); + assert_eq!(StorageEngineType::from_config_name("ssd"), Ssd); + assert_eq!(StorageEngineType::from_config_name("kRocksDB"), Ssd); + assert_eq!(StorageEngineType::from_config_name("pmem"), Pmem); + assert_eq!(StorageEngineType::from_config_name("multi_ssd"), MultiSsd); + assert_eq!(StorageEngineType::from_config_name("simple"), Simple); + assert_eq!(StorageEngineType::from_config_name("dram"), Dram); + assert!(Ssd.is_ssd_like()); + assert!(MultiSsd.is_ssd_like()); + assert!(!Dram.is_ssd_like()); + assert_eq!(Pmem.canonical_instance_type(), CacheInstanceType::Pmem); + assert_eq!(Dram.canonical_instance_type(), CacheInstanceType::Dram); // unknown code and name fall back to the default engine - assert_eq!(StorageEngineType::from_reference_code(200), kDRAM); - assert_eq!(StorageEngineType::from_reference_name("not-a-real-engine"), kDRAM); + assert_eq!(StorageEngineType::from_config_code(200), Dram); + assert_eq!(StorageEngineType::from_config_name("not-a-real-engine"), Dram); } #[test] @@ -7373,10 +7511,10 @@ mod tests { index.ScanIndexForRecover(|_key, _value| scanned += 1); assert!(scanned >= 1); - let mut wb = WriteBuffer::new(WriteBufferType::kUserDataBuf, 1024); + let mut wb = WriteBuffer::new(WriteBufferType::UserDataBuf, 1024); assert_eq!(wb.Capacity(), 1024); assert_eq!(wb.Count(), 0); - assert_eq!(wb.BufType(), WriteBufferType::kUserDataBuf); + assert_eq!(wb.BufType(), WriteBufferType::UserDataBuf); wb.PushBack("k1", b"v1".to_vec()); wb.PushBack("k2", b"v22".to_vec()); assert_eq!(wb.Count(), 2); @@ -7390,16 +7528,16 @@ mod tests { #[test] fn rdma_utils_and_policy_conversions() { assert_eq!( - RdmaReplacementPolicyType::FIFO.as_replacement_policy_type(), - ReplacementPolicyType::kFIFO + RdmaReplacementPolicyType::Fifo.as_replacement_policy_type(), + ReplacementPolicyType::Fifo ); assert_eq!( - RdmaReplacementPolicyType::LRU.as_replacement_policy_type(), - ReplacementPolicyType::kLRU + RdmaReplacementPolicyType::Lru.as_replacement_policy_type(), + ReplacementPolicyType::Lru ); assert_eq!( - RdmaReplacementPolicyType::OTHER.as_replacement_policy_type(), - ReplacementPolicyType::kMaxCode + RdmaReplacementPolicyType::Other.as_replacement_policy_type(), + ReplacementPolicyType::MaxCode ); let mut generator = RandomStringGenerator::new(); @@ -7419,44 +7557,44 @@ mod tests { CacheAccessRecordType::Get, CacheAccessRecordType::Delete, ] { - assert_eq!(CacheAccessRecordType::from_reference_code(ty.reference_code()), Some(ty)); + assert_eq!(CacheAccessRecordType::from_config_code(ty.config_code()), Some(ty)); } - assert_eq!(CacheAccessRecordType::from_reference_code(0), None); + assert_eq!(CacheAccessRecordType::from_config_code(0), None); assert_eq!( - CacheAccessRecordType::from_reference_code(CacheAccessRecordType::kMaxCode), + CacheAccessRecordType::from_config_code(CacheAccessRecordType::kMaxCode), None ); // CacheDataPlacement name parsing (fallible) assert_eq!( - CacheDataPlacement::try_from_reference_name("SideBySide").unwrap(), + CacheDataPlacement::try_from_config_name("SideBySide").unwrap(), CacheDataPlacement::SideBySide ); assert_eq!( - CacheDataPlacement::try_from_reference_name("Tiered").unwrap(), + CacheDataPlacement::try_from_config_name("Tiered").unwrap(), CacheDataPlacement::Tiered ); - assert!(CacheDataPlacement::try_from_reference_name("nonsense").is_err()); + assert!(CacheDataPlacement::try_from_config_name("nonsense").is_err()); // DRAMPMEMDataPlacementType conversions to/from CacheDataPlacement and names assert_eq!( DRAMPMEMDataPlacementType::from_cache_data_placement(CacheDataPlacement::SideBySide), - DRAMPMEMDataPlacementType::kSideBySide + DRAMPMEMDataPlacementType::SideBySide ); assert_eq!( DRAMPMEMDataPlacementType::from_cache_data_placement(CacheDataPlacement::Tiered), - DRAMPMEMDataPlacementType::kTiered + DRAMPMEMDataPlacementType::Tiered ); assert_eq!( - DRAMPMEMDataPlacementType::kTiered.as_cache_data_placement(), + DRAMPMEMDataPlacementType::Tiered.as_cache_data_placement(), CacheDataPlacement::Tiered ); for ty in [ - DRAMPMEMDataPlacementType::kSideBySide, - DRAMPMEMDataPlacementType::kTiered, - DRAMPMEMDataPlacementType::kMaxCode, + DRAMPMEMDataPlacementType::SideBySide, + DRAMPMEMDataPlacementType::Tiered, + DRAMPMEMDataPlacementType::MaxCode, ] { - assert!(!ty.as_reference_name().is_empty()); + assert!(!ty.as_config_name().is_empty()); } } @@ -7483,8 +7621,8 @@ mod tests { cache.put_memory_only(k("mem"), b"m".to_vec()); assert_eq!(cache.get_memory(&k("mem")), Some(b"m".to_vec())); - assert!(cache.get_capacity(CacheInstanceType::kDRAM) > 0); - let _ = cache.get_used(CacheInstanceType::kDRAM); + assert!(cache.get_capacity(CacheInstanceType::Dram) > 0); + let _ = cache.get_used(CacheInstanceType::Dram); } #[test] @@ -7510,8 +7648,8 @@ mod tests { // introspection is callable let _ = cache.used_space_for_tier(CacheTier::Memory); - let _ = cache.get_used(CacheInstanceType::kDRAM); - let _ = cache.get_replacement_policy_type(CacheInstanceType::kDRAM); + let _ = cache.get_used(CacheInstanceType::Dram); + let _ = cache.get_replacement_policy_type(CacheInstanceType::Dram); let _ = cache.replacement_policy_for_tier(CacheTier::Memory); cache.reset().unwrap(); @@ -7553,16 +7691,16 @@ mod tests { mgr.SetWriteEnabled(false); assert!(!mgr.WriteEnabled()); mgr.Start(); - let _ = mgr.put(("k1", b"v1".to_vec()), WriteBufferType::kUserDataBuf); - let _ = mgr.put(("k2", b"v2".to_vec()), WriteBufferType::kUserDataBuf); - let _ = mgr.buffered_count(WriteBufferType::kUserDataBuf); + let _ = mgr.put(("k1", b"v1".to_vec()), WriteBufferType::UserDataBuf); + let _ = mgr.put(("k2", b"v2".to_vec()), WriteBufferType::UserDataBuf); + let _ = mgr.buffered_count(WriteBufferType::UserDataBuf); let _ = mgr.FlushBuffers(); let _ = mgr.flushed_records(); mgr.Stop(); let _ = BufferManager::new().capacity_per_buf(); let mut mgr2 = BufferManager::with_config(1024, 0.5, 512); - let mut wb = WriteBuffer::new(WriteBufferType::kUserDataBuf, 1024); + let mut wb = WriteBuffer::new(WriteBufferType::UserDataBuf, 1024); wb.PushBack("x", b"y".to_vec()); assert_eq!(mgr2.flush_buffer(wb), 1); } @@ -7631,12 +7769,12 @@ mod tests { assert_eq!(decode_colored_ptr(mask_colored_ptr_size(0, 3)).0, 3); let _ = MaskColoredPtrLBA(0, 4); let _ = MaskColoredPtrMemoryAddress(0, 0x10); - let _ = MaskColoredPtrRecordState(0, RecordStateType::kNormal); - let _ = mask_colored_ptr_record_state(0, RecordStateType::kPinned); + let _ = MaskColoredPtrRecordState(0, RecordStateType::Normal); + let _ = mask_colored_ptr_record_state(0, RecordStateType::Pinned); // BufferEncoder size calculators let encoder = BufferEncoder::new(4096); - let mut wb = WriteBuffer::new(WriteBufferType::kUserDataBuf, 1024); + let mut wb = WriteBuffer::new(WriteBufferType::UserDataBuf, 1024); wb.PushBack("k", b"v".to_vec()); let _ = encoder.calculate_encoded_data_size(&wb); let _ = encoder.calculate_encoded_oplog_size(&wb); @@ -7708,8 +7846,8 @@ mod tests { #[test] fn rdma_storage_engine_ops() { - let mut engine = RdmaStorageEngine::new(RdmaStorageEngineType::DRAM, 1 << 20); - assert!(matches!(engine.storage_type(), RdmaStorageEngineType::DRAM)); + let mut engine = RdmaStorageEngine::new(RdmaStorageEngineType::Dram, 1 << 20); + assert!(matches!(engine.storage_type(), RdmaStorageEngineType::Dram)); assert_eq!(engine.capacity(), 1 << 20); assert_eq!(engine.used(), 0); let ptr = engine.Put(b"key", b"value").expect("rdma put allocates"); @@ -7811,7 +7949,7 @@ mod tests { let _ = MatrixCacheBuilder::build_concurrent_simple_lru_cache(1024); let _ = MatrixCacheBuilder::build_memcached_wrapper(1024); - // DRAM-only options avoid needing an on-disk SSD tier + // Dram-only options avoid needing an on-disk Ssd tier let opts = || CacheOptions::new(1 << 16, 0, 0); let cache = MatrixCacheBuilder::build_cache(opts()); cache.put(CacheKey::string(0, "a"), b"1".to_vec()).unwrap(); @@ -7870,8 +8008,8 @@ mod tests { cache.release(h); } - cache.set_capacity_for_instance(CacheInstanceType::kDRAM, 2 << 20); - cache.set_replacement_policy_type(CacheInstanceType::kDRAM, CacheReplacementPolicy::Fifo); + cache.set_capacity_for_instance(CacheInstanceType::Dram, 2 << 20); + cache.set_replacement_policy_type(CacheInstanceType::Dram, CacheReplacementPolicy::Fifo); let policy = cache.production_tiering_policy(); cache.update_production_tiering_policy(policy); } @@ -7890,7 +8028,7 @@ mod tests { let mut table = RdmaHashTable::>::new(16); let key = b"hkey".to_vec(); assert!(table.Get(&key).addr.is_none()); - let _ = table.Put(key.clone(), 0x1000, 5, RdmaStorageEngineType::DRAM); + let _ = table.Put(key.clone(), 0x1000, 5, RdmaStorageEngineType::Dram); let _ = table.Get(&key); let _ = table.Del(&key); let _ = table.get_bucket(0); @@ -7898,14 +8036,14 @@ mod tests { } #[test] - fn parity_alloc_utils_parse_allocate_persist_thread_ids_and_pmem_files() { - assert_eq!(ParseAllocatorType("Log"), AllocatorType::kLogBasedAllocator); + fn alloc_utils_parse_allocate_persist_thread_ids_and_pmem_files() { + assert_eq!(ParseAllocatorType("Log"), AllocatorType::LogBasedAllocator); assert_eq!( ParseAllocatorType("Pool"), - AllocatorType::kPoolBasedAllocator + AllocatorType::PoolBasedAllocator ); - assert_eq!(ParseAllocatorType("Jemalloc"), AllocatorType::kJeAllocator); - assert_eq!(ParseAllocatorType("missing"), AllocatorType::kMaxCode); + assert_eq!(ParseAllocatorType("Jemalloc"), AllocatorType::JeAllocator); + assert_eq!(ParseAllocatorType("missing"), AllocatorType::MaxCode); let ptr = DramAllocateObject(4096, 4096).unwrap(); assert_eq!(ptr % 4096, 0); @@ -7951,7 +8089,7 @@ mod tests { } #[test] - fn parity_simple_log_based_allocator_allocates_seals_frees_and_reports_stats() { + fn simple_log_based_allocator_allocates_seals_frees_and_reports_stats() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(16); let ptr = allocator.Allocate(8).unwrap(); @@ -7997,7 +8135,7 @@ mod tests { } #[test] - fn parity_simple_log_based_allocator_supports_trait_consumers() { + fn simple_log_based_allocator_supports_trait_consumers() { fn allocate_and_seal( allocator: &mut A, ) -> Result { @@ -8019,7 +8157,7 @@ mod tests { } #[test] - fn parity_storage_gc_controller_lifecycle_pause_and_force_gc_match_surface() { + fn storage_gc_controller_lifecycle_pause_and_force_gc_match_surface() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); let ptr = allocator.Allocate(8).unwrap(); allocator.write(ptr, b"gc-ready").unwrap(); @@ -8050,7 +8188,7 @@ mod tests { } #[test] - fn parity_storage_gc_controller_poll_paces_collection_checks() { + fn storage_gc_controller_poll_paces_collection_checks() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); let first = allocator.Allocate(8).unwrap(); allocator.Free(first, 8).unwrap(); @@ -8094,7 +8232,7 @@ mod tests { } #[test] - fn parity_storage_gc_controller_respects_enable_gate_and_manual_gc_job() { + fn storage_gc_controller_respects_enable_gate_and_manual_gc_job() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(64); let ptr_a = allocator.Allocate(4).unwrap(); let ptr_b = allocator.Allocate(4).unwrap(); @@ -8118,7 +8256,7 @@ mod tests { } #[test] - fn parity_cache_executor_reuses_common_and_gc_executors_and_runs_tasks() { + fn cache_executor_reuses_common_and_gc_executors_and_runs_tasks() { CacheExecutor::DestroyAllExecutors(); CacheExecutor::Configure(CacheExecutorConfig { common_executor_num_threads: 3, @@ -8148,7 +8286,7 @@ mod tests { } #[test] - fn parity_cache_executor_creates_pmem_numa_executors_and_destroy_resets() { + fn cache_executor_creates_pmem_numa_executors_and_destroy_resets() { CacheExecutor::DestroyAllExecutors(); CacheExecutor::Configure(CacheExecutorConfig { common_executor_num_threads: 1, @@ -8178,7 +8316,7 @@ mod tests { } #[test] - fn parity_async_writer_runs_write_then_callback_and_tracks_counters() { + fn async_writer_runs_write_then_callback_and_tracks_counters() { let allocator = SimpleLogBasedMemoryAllocator::with_capacity(128); let mut writer = AsyncWriter::new(allocator); @@ -8208,7 +8346,7 @@ mod tests { } #[test] - fn parity_async_writer_preserves_addr_and_stop_rejects_new_writes() { + fn async_writer_preserves_addr_and_stop_rejects_new_writes() { let mut allocator = SimpleLogBasedMemoryAllocator::with_capacity(128); let existing = allocator.Allocate(4).unwrap(); allocator.write(existing, b"seed").unwrap(); @@ -8251,13 +8389,13 @@ mod tests { } #[test] - fn parity_pmem_dispatcher_round_robins_put_tasks_across_numa_writers() { + fn pmem_dispatcher_round_robins_put_tasks_across_numa_writers() { let mut dispatcher = PMemDispatcher::new(2, 128); assert!(dispatcher.Start()); assert_eq!(dispatcher.numa_count(), 2); assert_eq!( dispatcher.allocator_type(), - AllocatorType::kLogBasedAllocator + AllocatorType::LogBasedAllocator ); for (key, value) in [("first", b"one".to_vec()), ("second", b"two".to_vec())] { @@ -8292,7 +8430,7 @@ mod tests { } #[test] - fn parity_pmem_dispatcher_routes_addr_tasks_to_owner_numa() { + fn pmem_dispatcher_routes_addr_tasks_to_owner_numa() { let mut alloc0 = SimpleLogBasedMemoryAllocator::with_capacity_and_base(128, 1 << 48); let mut alloc1 = SimpleLogBasedMemoryAllocator::with_capacity_and_base(128, 2 << 48); let ptr0 = alloc0.Allocate(4).unwrap(); @@ -8327,7 +8465,7 @@ mod tests { } #[test] - fn parity_pmem_dispatcher_supports_test_allocator_access_and_stop() { + fn pmem_dispatcher_supports_test_allocator_access_and_stop() { let mut dispatcher = PMemDispatcher::new(2, 128); dispatcher.Start(); @@ -8368,7 +8506,51 @@ mod tests { } #[test] - fn parity_replacement_fifo_evicts_oldest_and_invokes_handler() { + fn replacement_policies_stay_usable_after_reset() { + // A successful reset empties the index; it does not retire the + // policy. Asserting only that the post-reset put reports no + // evictions would not catch a regression here, because a policy + // that silently discards the buffer reports no evictions too. + // Check the buffer actually landed. + let mut fifo = ReplacementFIFO::new(1 << 20); + fifo.Init().unwrap(); + fifo.Put(test_buffer("before", b"1")); + assert!(fifo.GetUsedSpace() > 0); + + fifo.Reset().unwrap(); + assert_eq!(fifo.GetUsedSpace(), 0); + assert_eq!(fifo.GetItemNum(), 0); + assert!(fifo.Get("before").is_none()); + + assert!(fifo.Put(test_buffer("after", b"2")).is_empty()); + assert_eq!( + fifo.Peek("after").map(|buffer| buffer.Data().to_vec()), + Some(b"2".to_vec()), + "a reset fifo must still accept buffers" + ); + assert!(fifo.GetUsedSpace() > 0); + + let mut slru = ReplacementSLRU::new(1 << 20); + slru.Init().unwrap(); + slru.Put(test_buffer("before", b"1")); + assert!(slru.GetUsedSpace() > 0); + + slru.Reset().unwrap(); + assert_eq!(slru.GetUsedSpace(), 0); + assert_eq!(slru.GetItemNum(), 0); + assert!(slru.Get("before").is_none()); + + assert!(slru.Put(test_buffer("after", b"2")).is_empty()); + assert_eq!( + slru.Peek("after").map(|buffer| buffer.Data().to_vec()), + Some(b"2".to_vec()), + "a reset slru must still accept buffers" + ); + assert!(slru.GetUsedSpace() > 0); + } + + #[test] + fn replacement_fifo_evicts_oldest_and_invokes_handler() { let mut fifo = ReplacementFIFO::new(5); fifo.Init().unwrap(); let evicted_keys = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -8393,7 +8575,7 @@ mod tests { } #[test] - fn parity_replacement_fifo_update_guards_raw_data_and_preserves_order() { + fn replacement_fifo_update_guards_raw_data_and_preserves_order() { let mut fifo = ReplacementFIFO::new(32); fifo.Init().unwrap(); fifo.Put(test_buffer("guarded", b"old")); @@ -8413,7 +8595,7 @@ mod tests { } #[test] - fn parity_replacement_fifo_overwrite_keeps_original_queue_position() { + fn replacement_fifo_overwrite_keeps_original_queue_position() { let mut fifo = ReplacementFIFO::new(6); fifo.Init().unwrap(); // Three 2-byte entries exactly fill the policy. @@ -8440,7 +8622,7 @@ mod tests { } #[test] - fn parity_replacement_fifo_delete_leaves_no_queue_tombstone() { + fn replacement_fifo_delete_leaves_no_queue_tombstone() { let mut fifo = ReplacementFIFO::new(1 << 12); fifo.Init().unwrap(); for index in 0..256 { @@ -8466,7 +8648,7 @@ mod tests { } #[test] - fn parity_replacement_slru_get_records_access_without_reordering() { + fn replacement_slru_get_records_access_without_reordering() { let mut slru = ReplacementSLRU::with_num_segments(100, 1); slru.Init().unwrap(); slru.TEST_ConfigLRUMaintainer(false); @@ -8532,7 +8714,7 @@ mod tests { } #[test] - fn parity_replacement_slru_tracks_hot_warm_cold_and_fetch_flags() { + fn replacement_slru_tracks_hot_warm_cold_and_fetch_flags() { let mut slru = ReplacementSLRU::new(6); slru.Init().unwrap(); slru.TEST_ConfigLRUMaintainer(false); @@ -8555,7 +8737,7 @@ mod tests { } #[test] - fn parity_replacement_slru_update_delete_and_capacity_shrink() { + fn replacement_slru_update_delete_and_capacity_shrink() { let mut slru = ReplacementSLRU::new(32); slru.Init().unwrap(); slru.Put(test_buffer("x", b"old")); @@ -8577,7 +8759,7 @@ mod tests { } #[test] - fn parity_replacement_slru_resolves_segment_count_from_capacity_and_request() { + fn replacement_slru_resolves_segment_count_from_capacity_and_request() { // A capacity smaller than the segment count collapses to one segment, // otherwise every segment would get a zero byte budget. assert_eq!(ReplacementSLRU::new(6).num_segments(), 1); @@ -8603,7 +8785,7 @@ mod tests { } #[test] - fn parity_replacement_slru_shards_keys_and_bounds_each_segment() { + fn replacement_slru_shards_keys_and_bounds_each_segment() { let mut slru = ReplacementSLRU::new(1 << 16); slru.Init().unwrap(); assert_eq!(slru.num_segments(), SLRU_DEFAULT_NUM_SEGMENTS); @@ -8647,7 +8829,7 @@ mod tests { } #[test] - fn parity_replacement_slru_accounting_survives_overwrite_delete_and_reuse() { + fn replacement_slru_accounting_survives_overwrite_delete_and_reuse() { let mut slru = ReplacementSLRU::with_num_segments(1 << 14, 4); slru.Init().unwrap(); @@ -8690,7 +8872,7 @@ mod tests { } #[test] - fn parity_replacement_slru_maintainer_promotes_active_and_demotes_untouched() { + fn replacement_slru_maintainer_promotes_active_and_demotes_untouched() { let mut slru = ReplacementSLRU::with_num_segments(100, 1); slru.Init().unwrap(); slru.TEST_ConfigLRUMaintainer(false); @@ -8744,7 +8926,7 @@ mod tests { } #[test] - fn parity_replacement_slru_maintainer_evicts_cold_tail_over_segment_budget() { + fn replacement_slru_maintainer_evicts_cold_tail_over_segment_budget() { let mut slru = ReplacementSLRU::with_num_segments(40, 1); slru.Init().unwrap(); // Give hot and warm no share of the budget, so every insert is demoted @@ -8786,7 +8968,7 @@ mod tests { } #[test] - fn parity_concurrent_slru_matches_the_single_threaded_segment_layout() { + fn concurrent_slru_matches_the_single_threaded_segment_layout() { let concurrent = ConcurrentReplacementSLRU::with_num_segments(1 << 16, 256); let single = ReplacementSLRU::with_num_segments(1 << 16, 256); assert_eq!(concurrent.num_segments(), single.num_segments()); @@ -8805,7 +8987,7 @@ mod tests { } #[test] - fn parity_concurrent_slru_serves_threads_through_per_segment_locks() { + fn concurrent_slru_serves_threads_through_per_segment_locks() { let policy = ConcurrentReplacementSLRU::with_num_segments(1 << 16, 64); policy.Init().unwrap(); @@ -8844,7 +9026,7 @@ mod tests { } #[test] - fn parity_concurrent_slru_reports_evictions_from_every_segment() { + fn concurrent_slru_reports_evictions_from_every_segment() { let policy = ConcurrentReplacementSLRU::with_num_segments(256, 4); policy.Init().unwrap(); let evicted = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -8867,7 +9049,7 @@ mod tests { } #[test] - fn parity_concurrent_slru_round_trips_values_and_maintainer_passes() { + fn concurrent_slru_round_trips_values_and_maintainer_passes() { let policy = ConcurrentReplacementSLRU::with_num_segments(1 << 14, 8); policy.Init().unwrap(); policy.Put(test_buffer("round-trip", b"value")); @@ -8919,7 +9101,7 @@ mod tests { // A hit moves the key to the back and never duplicates it, which is // what a plain deque needs a full rescan to guarantee. - assert!(order.touch(&order_key(0))); + assert!(order.move_to_back(&order_key(0))); assert_eq!(order.back(), Some(&order_key(0))); assert_eq!(order.len(), 4); assert_eq!( @@ -8928,14 +9110,14 @@ mod tests { ); // Touching the key that is already most recent is a no-op. - assert!(order.touch(&order_key(0))); + assert!(order.move_to_back(&order_key(0))); assert_eq!( order_keys(&order), vec![order_key(1), order_key(2), order_key(3), order_key(0)] ); // Touching an absent key reports it and changes nothing. - assert!(!order.touch(&order_key(99))); + assert!(!order.move_to_back(&order_key(99))); assert_eq!(order.len(), 4); // Eviction takes the least recently used first. @@ -8986,6 +9168,44 @@ mod tests { assert_eq!(empty.iter_rev().count(), 0); } + #[test] + fn zero_copy_lru_keeps_counting_removed_but_pinned_bytes() { + let cache = ZeroCopySimpleLRUCache::new(4 * 64); + let key = CacheKey::string(0, "pinned-entry"); + let handle = cache + .InsertPinned(key.clone(), vec![118u8; 32], 64) + .unwrap() + .expect("pinned handle"); + assert_eq!(cache.Size(), 64); + + // The entry leaves the index, but a handle still holds the value, so + // those bytes are still resident and must keep counting. Releasing + // them here would let the cache admit data it has no room for. + cache.Remove(&key).unwrap(); + assert!(cache.Lookup(&key).unwrap().is_none()); + assert_eq!( + cache.Size(), + 64, + "a removed entry that is still pinned must stay accounted" + ); + + // Dropping the last pin is what actually frees the space. + cache.Release(handle); + assert_eq!(cache.Size(), 0); + } + + #[test] + fn zero_copy_lru_frees_removed_bytes_when_no_handle_holds_them() { + let cache = ZeroCopySimpleLRUCache::new(4 * 64); + let key = CacheKey::string(0, "unpinned-entry"); + cache.Insert(key.clone(), vec![118u8; 32], 64).unwrap(); + assert_eq!(cache.Size(), 64); + + // Nothing holds this one, so removal frees it immediately. + cache.Remove(&key).unwrap(); + assert_eq!(cache.Size(), 0); + } + #[test] fn simple_lru_evicts_the_coldest_entry_first() { let cache = SimpleLRUCache::new(3 * 64); @@ -9052,9 +9272,9 @@ mod tests { deque.retain(|candidate| candidate != &key); deque.push_back(key.clone()); } - assert!(order.touch(&key)); + assert!(order.move_to_back(&key)); } else { - assert!(!order.touch(&key)); + assert!(!order.move_to_back(&key)); } } 3 => { @@ -9096,7 +9316,7 @@ mod tests { } #[test] - fn parity_hash_uint64_matches_matrixcache_vectors() { + fn hash_uint64_matches_matrixcache_vectors() { assert_eq!(hash_uint64(0), 0x5b03_af84_387a_42c6); assert_eq!(hash_uint64(1), 0xa13a_3e40_1240_2345); assert_eq!(hash_uint64(2), 0xcd41_43fa_e38a_71fe); @@ -9106,7 +9326,7 @@ mod tests { } #[test] - fn parity_murmur_hash2_matches_matrixcache_vectors() { + fn murmur_hash2_matches_matrixcache_vectors() { assert_eq!(mur_mur_hash2(b""), 0xca88_1466); assert_eq!(mur_mur_hash2(b"a"), 0xe94e_6ebd); assert_eq!(mur_mur_hash2(b"abc"), 0x6d5e_3568); @@ -9131,7 +9351,7 @@ mod tests { } #[test] - fn parity_tools_utils_random_and_hashed_key_helpers_match_surface() { + fn tools_utils_random_and_hashed_key_helpers_match_surface() { assert_eq!(xxh32_with_seed(b"", 0), 0x02cc_5d05); assert_eq!(xxh32_with_seed(b"hello", 0), 0xfb00_77f9); @@ -9157,7 +9377,7 @@ mod tests { } #[test] - fn parity_round_up_matches_align_util_macro_semantics() { + fn round_up_matches_align_util_macro_semantics() { assert_eq!(round_up(0, 8), 0); assert_eq!(round_up(1, 8), 8); assert_eq!(round_up(8, 8), 8); @@ -9168,7 +9388,7 @@ mod tests { } #[test] - fn parity_numa_info_exposes_stable_single_node_topology() { + fn numa_info_exposes_stable_single_node_topology() { NumaInfo::Init(); assert!(NumaInfo::GetNumAllCores() >= 1); assert!(NumaInfo::GetNumOnlineCores() >= NumaInfo::GetNumAllCores()); @@ -9190,8 +9410,8 @@ mod tests { assert_eq!(cache.shard_count(), 4); assert_eq!(cache.capacity_for_tier(CacheTier::Memory), 96); assert_eq!(cache.CapacityForTier(CacheTier::Ssd), 4096); - assert_eq!(cache.GetCapacity(CacheInstanceType::kDRAM), 96); - assert_eq!(cache.GetCapacity(CacheInstanceType::kSSD), 4096); + assert_eq!(cache.GetCapacity(CacheInstanceType::Dram), 96); + assert_eq!(cache.GetCapacity(CacheInstanceType::Ssd), 4096); let config_cache = MatrixCacheBuilder::build_sharded_cache(CacheOptions::new(32, 32, 0), 2); assert!(config_cache.stop()); config_cache @@ -9231,8 +9451,8 @@ mod tests { assert!(latency.put_count >= keys.len() as u64); assert!(latency.get_count >= keys.len() as u64); assert!(latency.histogram_ready); - assert!(cache.GetUsed(CacheInstanceType::kDRAM) > 0); - assert!(cache.GetUsed(CacheInstanceType::kSSD) > 0); + assert!(cache.GetUsed(CacheInstanceType::Dram) > 0); + assert!(cache.GetUsed(CacheInstanceType::Ssd) > 0); let repeated = keys[3].clone(); let other = keys[7].clone(); @@ -9262,9 +9482,9 @@ mod tests { cache.ReplacementPolicyForTier(CacheTier::Memory), CacheReplacementPolicy::WeightedHotnessLru ); - cache.SetReplacementPolicyType(CacheInstanceType::kSSD, CacheReplacementPolicy::Fifo); + cache.SetReplacementPolicyType(CacheInstanceType::Ssd, CacheReplacementPolicy::Fifo); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kSSD), + cache.GetReplacementPolicyType(CacheInstanceType::Ssd), CacheReplacementPolicy::Fifo ); assert_eq!( @@ -9279,33 +9499,33 @@ mod tests { let eviction = cache.EvictionReport(); assert!(eviction.memory_capacity_evictions > 0); assert!(eviction.memory_slot_evictions > 0); - cache.SetCapacityForInstance(CacheInstanceType::kSSD, 1024); - assert_eq!(cache.GetCapacity(CacheInstanceType::kSSD), 1024); + cache.SetCapacityForInstance(CacheInstanceType::Ssd, 1024); + assert_eq!(cache.GetCapacity(CacheInstanceType::Ssd), 1024); cache.SetReplacementPolicyForTier(CacheTier::Memory, CacheReplacementPolicy::Fifo); assert_eq!( - cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::Fifo ); let running_cache = MatrixCacheBuilder::build_sharded_cache(CacheOptions::new(32, 0, 0), 2); running_cache.start().unwrap(); assert!(matches!( running_cache.TrySetReplacementPolicyType( - CacheInstanceType::kDRAM, + CacheInstanceType::Dram, CacheReplacementPolicy::Fifo, ), Err(CacheError::AlreadyStarted) )); assert_eq!( - running_cache.GetReplacementPolicyType(CacheInstanceType::kDRAM), + running_cache.GetReplacementPolicyType(CacheInstanceType::Dram), CacheReplacementPolicy::WeightedHotnessLru ); assert!(matches!( cache.TrySetReplacementPolicyType( - CacheInstanceType::kUnified, + CacheInstanceType::Unified, CacheReplacementPolicy::Fifo, ), - Err(CacheError::UnsupportedInstance(CacheInstanceType::kUnified)) + Err(CacheError::UnsupportedInstance(CacheInstanceType::Unified)) )); assert!(matches!( cache.TrySetReplacementPolicyForTier(CacheTier::Reject, CacheReplacementPolicy::Fifo), @@ -9318,11 +9538,11 @@ mod tests { 2, ); assert_eq!( - api.capacity_for_instance_cache(CacheInstanceType::kDRAM), + api.capacity_for_instance_cache(CacheInstanceType::Dram), 64 ); assert_eq!( - api.capacity_for_instance_cache(CacheInstanceType::kSSD), + api.capacity_for_instance_cache(CacheInstanceType::Ssd), 1024 ); let trait_key = CacheKey::string(7, "trait-key"); @@ -9332,9 +9552,9 @@ mod tests { api.lookup_cache(&trait_key).unwrap().unwrap(), b"trait-value".to_vec() ); - assert!(api.used_cache(CacheInstanceType::kDRAM) > 0); - api.set_capacity_for_instance_cache(CacheInstanceType::kDRAM, 8); - assert_eq!(api.capacity_for_instance_cache(CacheInstanceType::kDRAM), 8); + assert!(api.used_cache(CacheInstanceType::Dram) > 0); + api.set_capacity_for_instance_cache(CacheInstanceType::Dram, 8); + assert_eq!(api.capacity_for_instance_cache(CacheInstanceType::Dram), 8); api.reset_cache().unwrap(); assert_eq!(api.lookup_cache(&trait_key).unwrap(), None); @@ -9504,7 +9724,7 @@ mod tests { cache .test_insert( - CacheInstanceType::kPMEM, + CacheInstanceType::Pmem, acquired.clone(), b"pmem".to_vec(), 4,