fix(heap): stranded large spans after adoption; perf: segment scrub, large cache (2.2.4)

corpus/linux-gates.sh

@@ -8,7 +8,9 @@
 # an infinite-recursion bug, and was briefly read as a pass.
 set -uo pipefail
 source ~/.cargo/env 2>/dev/null
-cd /mnt/c/Users/talmo/coding/rusty_alloc || exit 1
+# The repo this script lives in, wherever it is mounted (it was a hard-coded
+# /mnt/c path, which exited 1 before running anything on any other layout).
+cd "$(cd "$(dirname "$0")/.." && pwd)" || exit 1
 
 # CLIPPY RUNS HERE TOO, and that is not redundant with the Windows gate.
 # Lints are platform-dependent: `c_long` is i64 on LP64 unix and i32 on

crates/rusty_alloc/CHANGELOG.md

@@ -7,6 +7,52 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- **Freed model memory no longer stays stranded after a loader thread
+  exits.** A thread that exits abandons its segments with their blocks live;
+  blocks freed into them later are collected when another thread adopts the
+  segment, but adoption retired only ONE dead large span per segment and left
+  the rest carved for the life of the process. A model loaded on one thread
+  and dropped on another kept ~0.6 of its size resident and unreusable,
+  whatever `purge_delay` said. Every dead large span is now retired. Adoption
+  also no longer releases the empty segment it has just parked (which left
+  the heap releasing every empty segment instead of keeping one).
+- **A freed huge block (> 32 MiB) is purged when purging is on.** It used to
+  rejoin the arena resident whatever `purge_delay` said: a dead Whisper-small
+  embedding (152 MiB) stayed resident for the life of the process. Details,
+  measurements and the consumer report: `docs/plans/huge-free-retention.md`.
+
+- **A deferred-free hook that allocates is no longer undefined behaviour.**
+  `mi_register_deferred_free`'s hook fired inside the generic allocation
+  path while that path held `&mut Heap`, so a hook that allocated (or freed)
+  re-entered the heap under a protected exclusive borrow. Miri reported it
+  (Stacked Borrows, `tests/openheimer.rs`), and it had kept the `miri` CI job
+  red since 2.2.3. The hook now fires at the next allocation entry, before
+  any `&mut Heap` exists. Cost: one flag test per slow-path entry (perl and
+  sqlite allocator instructions +0.16 % / +0.22 %).
+
+### Performance
+
+- **Recycled segments are no longer scrubbed whole.** Every thread's first
+  segment and every huge allocation zeroed the 45 KB header of a recycled
+  chunk; now only the owner table and the slots read before a carve are
+  zeroed, and a slot is zeroed at its first carve. Huge segments also skip
+  511 unused slot writes and scan one slot, not 511, when freed. Measured
+  (callgrind, memset counted as real instructions): 64 MiB malloc+free
+  8,959 -> 824 Ir (-90.8 %), a thread's life -12.4 %, a Rust thread-pool
+  workload -15.0 %.
+- **A freed large block (<= 4 MiB) is kept for same-size reuse** while
+  purging is off, and only when it is a lone free (never the end of a
+  teardown run, so a model swap lays out exactly as before). A 2 MiB
+  malloc/free pair 293 -> 138 Ir.
+- **Multi-chunk arena claims** read each bitmap word once and claim a
+  one-word run with one atomic: 6 -> 2 locked read-modify-writes per huge
+  allocation.
+
+Every opscan op is unchanged or better; perl, sqlite and Endless Sky are flat
+on allocator instructions; code size +361 bytes gzipped on wasm.
+
 ## [2.2.3](https://github.com/Remade-With-Rust/rusty_alloc/compare/rusty_alloc-v2.2.2...rusty_alloc-v2.2.3) - 2026-10-05
 
 ### Fixed

crates/rusty_alloc/src/alloc.rs

@@ -359,11 +359,74 @@ unsafe fn malloc_slow(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
         // only where a workload leaves populated free lists behind
         // (`mixed`, and `rptest` through `Heap::malloc`). A fast path is worth
         // its cost only where the thing it looks for is actually there.
-        return unsafe { (*(*hb).heap.get()).malloc_generic(size).0 };
+        //
+        // The deferred-free hook check stays in TAIL position too: a due hook
+        // jumps to the cold `malloc_slow_deferred`, which fires it and then
+        // allocates. Calling `fire_deferred_if_due` here instead put a call in
+        // this function and gave it a frame: big 91 -> 100, mixed +6.6 Ir/op.
+        // SAFETY: as above; the flag read forms no reference.
+        unsafe {
+            let h = (*hb).heap.get();
+            if (*h).deferred_due {
+                return malloc_slow_deferred(h, size);
+            }
+            return (*h).malloc_generic(size).0;
+        }
     }
     malloc_first(size)
 }
 
+/// [`malloc_slow`] when the heartbeat marked the deferred-free hook due: fire
+/// it while no `&mut Heap` exists, then allocate (see
+/// [`fire_deferred_if_due`]).
+///
+/// # Safety
+/// `h` is the calling thread's live, initialised heap.
+#[cold]
+#[inline(never)]
+unsafe fn malloc_slow_deferred(h: *mut crate::heap::Heap, size: usize) -> *mut u8 {
+    // SAFETY: per contract; the hook runs before `&mut Heap` is formed.
+    unsafe {
+        fire_deferred_if_due(h);
+        (*h).malloc_generic(size).0
+    }
+}
+
+/// Fire the deferred-free hook if the generic path's heartbeat marked it due
+/// ([`crate::heap::Heap::deferred_due`]). Called at every allocation entry
+/// BEFORE it forms `&mut Heap`, which is the point: run inside the heartbeat,
+/// a hook that allocates re-entered the heap while `malloc_generic`'s
+/// `&mut self` was protected — undefined behaviour under Stacked Borrows
+/// (Miri, `oh_f01_deferred_free_hook_that_mallocs_is_contained`, red on main
+/// since 2.2.3). Upstream calls the hook at the start of the generic path,
+/// with no aliasing rules to break; here it runs at the start of the NEXT
+/// allocation entry, which the hook's contract ("called occasionally, with a
+/// heartbeat") allows.
+///
+/// # Safety
+/// `h` is a live heap owned by the calling thread, and no reference to it is
+/// live.
+#[inline(always)]
+unsafe fn fire_deferred_if_due(h: *mut crate::heap::Heap) {
+    // SAFETY: a raw field read and write on the caller's own heap; no
+    // reference to the heap is formed, so the hook may re-enter it.
+    unsafe {
+        if (*h).deferred_due {
+            (*h).deferred_due = false;
+            fire_deferred_now();
+        }
+    }
+}
+
+/// The hook call itself, out of line and cold: it is an indirect call that is
+/// almost never reached, and inlined it would cost every entry's register
+/// allocation.
+#[cold]
+#[inline(never)]
+fn fire_deferred_now() {
+    crate::options::deferred_free(false);
+}
+
 /// A fresh thread's very first allocation: create the heap, then allocate.
 /// Split from [`malloc_slow`] so its register needs (keeping `size` alive
 /// across `init_thread_heap`) are paid once per THREAD, not once per slow
@@ -437,7 +500,11 @@ unsafe fn zalloc_slow(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
         return zalloc_first(size);
     }
     // SAFETY: a non-sentinel box is this thread's live, initialised box.
-    unsafe { (*(*hb).heap.get()).zalloc(size) }
+    unsafe {
+        let h = (*hb).heap.get();
+        fire_deferred_if_due(h);
+        (*h).zalloc(size)
+    }
 }
 
 /// A fresh thread's first zeroing allocation: create the heap, then allocate.
@@ -654,7 +721,10 @@ fn malloc_aligned_first(size: usize, align: usize, offset: usize) -> *mut u8 {
         return ptr::null_mut(); // heap creation failed: OOM ⇒ null
     }
     // SAFETY: own heap.
-    unsafe { (*h).malloc_aligned_at(size, align, offset).0 }
+    unsafe {
+        fire_deferred_if_due(h);
+        (*h).malloc_aligned_at(size, align, offset).0
+    }
 }
 
 /// `mi_zalloc_aligned`.
@@ -679,7 +749,9 @@ pub fn zalloc_aligned_at(size: usize, align: usize, offset: usize) -> *mut u8 {
     // SAFETY: a non-sentinel box is this thread's live, initialised box;
     // zero_block contract (zeroes [p, p+usable)).
     unsafe {
-        let (p, is_zero) = (*(*hb).heap.get()).malloc_aligned_at(size, align, offset);
+        let h = (*hb).heap.get();
+        fire_deferred_if_due(h);
+        let (p, is_zero) = (*h).malloc_aligned_at(size, align, offset);
         if !p.is_null() {
             zero_block(p, is_zero);
         }
@@ -697,6 +769,7 @@ fn zalloc_aligned_first(size: usize, align: usize, offset: usize) -> *mut u8 {
     }
     // SAFETY: own heap; zero_block contract.
     unsafe {
+        fire_deferred_if_due(h);
         let (p, is_zero) = (*h).malloc_aligned_at(size, align, offset);
         if !p.is_null() {
             zero_block(p, is_zero);
@@ -1570,7 +1643,11 @@ unsafe fn heap_of(hb: *mut init::HeapBox) -> *mut Heap {
 /// `hb` live and owned by the calling thread.
 pub unsafe fn heap_malloc(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
     // SAFETY: forwarded contract.
-    unsafe { (*heap_of(hb)).malloc(size).0 }
+    unsafe {
+        let h = heap_of(hb);
+        fire_deferred_if_due(h);
+        (*h).malloc(size).0
+    }
 }
 
 /// `mi_heap_zalloc`.
@@ -1580,7 +1657,9 @@ pub unsafe fn heap_malloc(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
 pub unsafe fn heap_zalloc(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
     // SAFETY: forwarded contract.
     unsafe {
-        let (p, is_zero) = (*heap_of(hb)).malloc(size);
+        let h = heap_of(hb);
+        fire_deferred_if_due(h);
+        let (p, is_zero) = (*h).malloc(size);
         if !p.is_null() {
             zero_block(p, is_zero);
         }
@@ -1599,7 +1678,11 @@ pub unsafe fn heap_malloc_aligned_at(
     offset: usize,
 ) -> *mut u8 {
     // SAFETY: forwarded contract.
-    unsafe { (*heap_of(hb)).malloc_aligned_at(size, align, offset).0 }
+    unsafe {
+        let h = heap_of(hb);
+        fire_deferred_if_due(h);
+        (*h).malloc_aligned_at(size, align, offset).0
+    }
 }
 
 /// `mi_heap_zalloc_aligned_at`.
@@ -1614,7 +1697,9 @@ pub unsafe fn heap_zalloc_aligned_at(
 ) -> *mut u8 {
     // SAFETY: forwarded contract.
     unsafe {
-        let (p, is_zero) = (*heap_of(hb)).malloc_aligned_at(size, align, offset);
+        let h = heap_of(hb);
+        fire_deferred_if_due(h);
+        let (p, is_zero) = (*h).malloc_aligned_at(size, align, offset);
         if !p.is_null() {
             zero_block(p, is_zero);
         }