From 870eaee0cc1843891d37354f53e18c7c372caa24 Mon Sep 17 00:00:00 2001 From: Martin Vit Date: Sun, 15 Nov 2026 03:22:17 +0000 Subject: [PATCH] Reclaim host prefix rows in least-recently-used order The DRAM prefix store treated the row of every non-root record with exactly one child as redundant. Once free retention space fell to two continuation rows, storing a record dropped the rows of all such parents at once. A single child does not show that the parent's row is unused. A request that branches from a checkpoint, such as a client side request or history the client renders differently from the generated tokens, stores a child that the original conversation never extends. The parent then keeps its pages but no longer matches lookups, so a conversation evicted from the GPU cannot be restored from host memory and prefills its whole prompt again. Ordering reclamation by use keeps the rows that requests reused most recently, which are the ones a returning conversation resumes from, without adding a retention rule or a cap. Core::thin_rows_for now reclaims queued redundant rows in ascending access order and stops once the free retention space exceeds the requested headroom. Rows it did not need stay queued. The redundancy test, pin handling, the headroom threshold, and leaf eviction are unchanged, and the disk store is not affected. docs/prefix-cache-offload.md states the order and its reason. The expected result is that a recently used parent keeps its row while older redundant rows exist, so a host lookup for the conversation it belongs to still finds a resume point. This is an expected benefit; it is not measured on a GPU service in this commit. Evidence. A Kimi K3.1 server with a 128 GiB DRAM store under agent traffic stored 4,369 records and dropped the rows of 4,070, and about 1,300 host lookups produced 16 restores; two requests of one conversation that had been idle for two and three minutes each prefilled more than 300,000 tokens again. Those figures were recorded before this change. The regression test pressure_keeps_the_row_of_a_recently_used_parent_whose_child_branched_away stores a chain, reuses its last checkpoint, and stores a branching child under pressure: it fails before the change and passes after it. The Selene library suite passes with 402 tests on Linux x86_64 with Rust 1.95.0. No GPU execution or service benchmark was run, and the effect on restore rates is unqualified. Signed-off-by: Martin Vit Design-Signed-Off-By: Claude (Fable 5.2); design-review=ed22 Co-Authored-By: Claude Fable 5.2 --- docs/prefix-cache-offload.md | 6 +++- selene/src/memory.rs | 56 +++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/docs/prefix-cache-offload.md b/docs/prefix-cache-offload.md index caf005839..fb010612e 100644 --- a/docs/prefix-cache-offload.md +++ b/docs/prefix-cache-offload.md @@ -278,7 +278,11 @@ pages. DRAM retains these redundant rows while its free retention space exceeds two continuation rows. Reserved transfer intake does not count as retention headroom. When that threshold is reached, or an incoming record needs more space than is free, DRAM reclaims eligible redundant rows before evicting other -records. Readers and spills delay row removal until their pins are released. +records. It reclaims them in least-recently-used order and stops once the +headroom is restored: a single child does not show that its parent's row is +unused, because a request that branches from the parent leaves that row as the +only resume point of the conversation it branched from. Readers and spills +delay row removal until their pins are released. DRAM releases row slots, disk truncates at the row segment, and later spills omit removed rows. A rewind or later fork restores its retained endpoint when available; otherwise it restores the nearest row-bearing ancestor and diff --git a/selene/src/memory.rs b/selene/src/memory.rs index 70907d7dd..4f7202867 100644 --- a/selene/src/memory.rs +++ b/selene/src/memory.rs @@ -720,14 +720,34 @@ impl Core { { return; } - for key in std::mem::take(&mut self.thinning) { + // A single child does not prove its parent's row useless: a request that + // branches from the parent (a side request, or history the client renders + // differently) leaves the parent as the only resume point of the main + // conversation. Reclaim the least recently used rows first, and only as + // many as the headroom needs, so recently used parents keep theirs. + let mut queued: Vec<_> = std::mem::take(&mut self.thinning) + .into_iter() + .filter_map(|key| self.index.node(&key).map(|node| (node.accessed(), key))) + .collect(); + queued.sort_unstable(); + let mut queued = queued.into_iter(); + for (_, key) in queued.by_ref() { if !self.index.row_redundant(&key, false) { continue; } if !self.thin(&key) { self.thinning.insert(key); } + if self + .pool + .free_slots() + .saturating_sub(self.pool.intake_slots()) + > headroom + { + break; + } } + self.thinning.extend(queued.map(|(_, key)| key)); } fn mark_durable(&mut self, key: &PrefixKey) -> Result<()> { @@ -2032,6 +2052,40 @@ mod tests { } } + #[test] + fn pressure_keeps_the_row_of_a_recently_used_parent_whose_child_branched_away() { + let mut core = store_core(4, 6, false); + let main = history(256, 1); + let root = published(&main, 64, None, 0, 1); + let older = published(&main, 128, Some(64), 0, 2); + let resume = published(&main, 192, Some(128), 0, 3); + for checkpoint in [&root, &older, &resume] { + assert!(core.store(checkpoint).unwrap()); + finish(&mut core); + } + // A side request reuses the main conversation's last checkpoint and + // then diverges, so its record cannot resume the main conversation. + core.reused(&resume.key); + let mut branch = PrefixHistory::new(IDENTITY); + let shared: Vec = (0..192u32) + .map(|t| t.wrapping_mul(2654435761).wrapping_add(1)) + .collect(); + branch.append(&shared, &[]).unwrap(); + branch + .append(&(0..64u32).map(|t| 7_000_000 + t).collect::>(), &[]) + .unwrap(); + let side = published(&branch, 256, Some(192), 0, 4); + assert!(core.store(&side).unwrap()); + finish(&mut core); + assert!(!core.index[&older.key].row, "the older redundant row is reclaimed first"); + assert!(core.index[&resume.key].row, "the recently used parent keeps its row"); + assert_eq!( + core.lookup(&main, 0, u64::MAX).map(|hit| hit.key), + Some(resume.key) + ); + assert_clean_slots(&core); + } + #[test] fn pressure_thinning_waits_for_the_last_row_pin() { let mut core = store_core(6, 1, false);