From f43ab12c4ac03da1854e2e6914bd3c9f43075959 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 7 Oct 2026 17:56:19 -0700 Subject: [PATCH 1/2] fix(heap): stranded large spans after adoption; perf: segment scrub, large cache Fixes (docs/plans/huge-free-retention.md, a consumer report): - adopt_segment retired ONE dead large span per adopted segment and left the rest carved forever (a large span is never queued, so no collect reaches it). A model loaded on a thread that then exited, and dropped on another, kept ~0.6 of its size resident whatever purge_delay said. Now every dead span is retired; the walk resumes at the merged span. Adoption also no longer re-counts and releases the empty segment that retire_span had just parked. - huge_free purges a dead huge block (> 32 MiB) before its chunks rejoin the arena when purging is on. It stayed resident: the consumer's unexplained ~150 MB was Whisper small's 152 MiB embedding. Performance (callgrind; memset counted as real instructions): - scrub_recycled: a recycled chunk zeroes the owner table and the slots read before a carve, not the whole 45 KB header; slots are zeroed at their first carve. Huge segments skip 511 strided slot writes and wait_no_remote_in_flight scans slot 1 only. 64 MiB malloc+free 8,959 -> 824 Ir; thread life -12.4 %; Rust thread-pool workload -15.0 %. - Per-heap large-span cache (<= 4 MiB, purging off, lone frees only, so a teardown run never leaves a span to be refilled out of order): 2 MiB malloc/free 293 -> 138 Ir. - Multi-chunk arena claim: one bitmap load per word, a one-word run claimed with one masked fetch_or. find_arena replaces three copies of the scan. Every opscan op unchanged or better; perl/sqlite/Endless Sky allocator Ir flat. Refuted and recorded in docs/opps.md: segment_map RMW batching (+15), masked chunk release (+9), flush-on-miss in large_alloc (+3 on every generic op). Validated: workspace --all-features, linux-gates (42 suites), embedded matrix 15/15, miri gate, semgrep, mimalloc-bench sweep 19/19, churn, real-world programs byte-identical across ra/mi/sys, alloc-eval checks on 8 game traces, downstream corpus (spacedb x3, rusty_alloc_default, rusty_zstd). Unsafe census 973 -> 977 (UNSAFE.md); wasm baseline 20321 -> 20873 B gz (deliberate). corpus/linux-gates.sh no longer hard-codes a /mnt/c path. Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/rust-globalalloc/Cargo.lock | 4 +- corpus/linux-gates.sh | 4 +- crates/rusty_alloc/CHANGELOG.md | 37 ++++ crates/rusty_alloc/UNSAFE.md | 6 +- crates/rusty_alloc/src/arena.rs | 180 ++++++++------- crates/rusty_alloc/src/heap.rs | 152 ++++++++++++- crates/rusty_alloc/src/segment.rs | 105 +++++++-- crates/rusty_alloc/src/segment_map.rs | 6 + crates/rusty_alloc/tests/adopt_large_spans.rs | 77 +++++++ crates/rusty_alloc/tests/double_free.rs | 48 ++++ crates/rusty_alloc/tests/huge_purge.rs | 43 ++++ crates/rusty_alloc/tests/large_cache.rs | 81 +++++++ crates/rusty_alloc/tests/spans.rs | 8 +- docs/opps.md | 166 ++++++++++++++ docs/plans/huge-free-retention.md | 209 ++++++++++++++++++ tools/unsafe-baseline.txt | 6 +- tools/wasm-size-baseline.txt | 2 +- 17 files changed, 1011 insertions(+), 123 deletions(-) create mode 100644 crates/rusty_alloc/tests/adopt_large_spans.rs create mode 100644 crates/rusty_alloc/tests/huge_purge.rs create mode 100644 crates/rusty_alloc/tests/large_cache.rs create mode 100644 docs/plans/huge-free-retention.md diff --git a/bench/rust-globalalloc/Cargo.lock b/bench/rust-globalalloc/Cargo.lock index 5e0516c..017717b 100644 --- a/bench/rust-globalalloc/Cargo.lock +++ b/bench/rust-globalalloc/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "rusty_alloc" -version = "2.2.0" +version = "2.2.3" dependencies = [ "libc", "portable-atomic", @@ -32,7 +32,7 @@ dependencies = [ [[package]] name = "rusty_alloc-api" -version = "2.2.0" +version = "2.2.3" dependencies = [ "rusty_alloc", ] diff --git a/corpus/linux-gates.sh b/corpus/linux-gates.sh index 46d9d5b..7c1239f 100644 --- a/corpus/linux-gates.sh +++ b/corpus/linux-gates.sh @@ -8,7 +8,9 @@ # an infinite-recursion bug, and was briefly read as a pass. set -uo pipefail source ~/.cargo/env 2>/dev/null -cd /mnt/c/Users/talmo/coding/rusty_alloc || exit 1 +# The repo this script lives in, wherever it is mounted (it was a hard-coded +# /mnt/c path, which exited 1 before running anything on any other layout). +cd "$(cd "$(dirname "$0")/.." && pwd)" || exit 1 # CLIPPY RUNS HERE TOO, and that is not redundant with the Windows gate. # Lints are platform-dependent: `c_long` is i64 on LP64 unix and i32 on diff --git a/crates/rusty_alloc/CHANGELOG.md b/crates/rusty_alloc/CHANGELOG.md index b4268cd..ac365a5 100644 --- a/crates/rusty_alloc/CHANGELOG.md +++ b/crates/rusty_alloc/CHANGELOG.md @@ -7,6 +7,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **Freed model memory no longer stays stranded after a loader thread + exits.** A thread that exits abandons its segments with their blocks live; + blocks freed into them later are collected when another thread adopts the + segment, but adoption retired only ONE dead large span per segment and left + the rest carved for the life of the process. A model loaded on one thread + and dropped on another kept ~0.6 of its size resident and unreusable, + whatever `purge_delay` said. Every dead large span is now retired. Adoption + also no longer releases the empty segment it has just parked (which left + the heap releasing every empty segment instead of keeping one). +- **A freed huge block (> 32 MiB) is purged when purging is on.** It used to + rejoin the arena resident whatever `purge_delay` said: a dead Whisper-small + embedding (152 MiB) stayed resident for the life of the process. Details, + measurements and the consumer report: `docs/plans/huge-free-retention.md`. + +### Performance + +- **Recycled segments are no longer scrubbed whole.** Every thread's first + segment and every huge allocation zeroed the 45 KB header of a recycled + chunk; now only the owner table and the slots read before a carve are + zeroed, and a slot is zeroed at its first carve. Huge segments also skip + 511 unused slot writes and scan one slot, not 511, when freed. Measured + (callgrind, memset counted as real instructions): 64 MiB malloc+free + 8,959 -> 824 Ir (-90.8 %), a thread's life -12.4 %, a Rust thread-pool + workload -15.0 %. +- **A freed large block (<= 4 MiB) is kept for same-size reuse** while + purging is off, and only when it is a lone free (never the end of a + teardown run, so a model swap lays out exactly as before). A 2 MiB + malloc/free pair 293 -> 138 Ir. +- **Multi-chunk arena claims** read each bitmap word once and claim a + one-word run with one atomic: 6 -> 2 locked read-modify-writes per huge + allocation. + +Every opscan op is unchanged or better; perl, sqlite and Endless Sky are flat +on allocator instructions; code size +361 bytes gzipped on wasm. + ## [2.2.3](https://github.com/Remade-With-Rust/rusty_alloc/compare/rusty_alloc-v2.2.2...rusty_alloc-v2.2.3) - 2026-10-05 ### Fixed diff --git a/crates/rusty_alloc/UNSAFE.md b/crates/rusty_alloc/UNSAFE.md index a99fbad..b8d8778 100644 --- a/crates/rusty_alloc/UNSAFE.md +++ b/crates/rusty_alloc/UNSAFE.md @@ -21,9 +21,9 @@ unsafe in DEPENDENCIES, and ours are `libc` plus bindings-only `windows-sys`. | Module | Count | What the `unsafe` is for | Last audit | |---|---:|---|---| | `alloc.rs` | 94 | The public entry points: raw-pointer reads on the malloc fast path (must never form `&mut` on the shared empty-heap sentinel), pointer-derived metadata on the free path (`segment_of`/`page_of`), block-content copies in the realloc family. **+15 on 2026-08-22**: the free path's two `asm!` sites — the memory-destination `used--` whose flags drive the retire branch (its `label` block is a separate item and carries its own `unsafe`), and the fused `cmp {tid}, fs:0` in both `free_inline` and `free_general` — plus `malloc_or`/`malloc_or_slow`, which give `operator new` a fast path whose miss is a tail call. Each asm reads or writes exactly one field it already had a valid pointer to; none widens what the surrounding code could already touch. **+2 on 2026-09-24 (instruction-count campaign, LEDGER):** `realloc` split into an inline null test and `unsafe fn realloc_live(p: NonNull, ..)` — one signature, and the `NonNull::new_unchecked` sits inside the block that already called it, on a pointer the line above proved non-null. The body is the old one unchanged; nothing new is dereferenced. **+10 on 2026-09-24 (round two, LEDGER):** (a) `realloc_move`, the moving arm shared by the growth and the shrink — one `unsafe fn`, its body block, and the two call-site blocks: the old inline move code, now reached with the copy length each arm already holds; (b) `zalloc` in `malloc`'s raw-read shape — its fast-path block reads through `hb`, which may be the sentinel, exactly as `malloc` does (no `&mut`, no write unless a block was popped, which proves the heap is real), and the cold `unsafe fn zalloc_slow` that performs the sentinel test the fast path dropped (signature + block); (c) `free_local_owned` / `free_local_no_xheap`, `owner_heap` split so its heap-creating fallback is a tail call — two signatures and their two blocks, the same loads and the same `free_local_at` as before; (d) `malloc_aligned_pow2`, an `unsafe fn` only because its caller must have proven the alignment a power of two (a violation is not memory-unsafe — the bound test still refuses a mask at or above half a segment). Nothing new is dereferenced anywhere; each carries its SAFETY line | 2026-09-24 | -| `heap.rs` | 77 | Owner-thread page/queue manipulation under raw pointers (no two `&mut Page` may coexist), the aligned-allocation peek, span carving. **+3 on 2026-08-19**: `try_unlink_huge_segment` split out of `remove_huge_segment`. **+15 on 2026-08-22**: `malloc_generic` split into a small entry plus `malloc_generic_walk`, with `grow_front`, `try_guarded` and `drain_delayed` as cold arms — each split adds an `unsafe fn` signature and its block while dereferencing nothing the single function did not — and the immortal `EMPTY_DELAYED` sentinel that lets the heartbeat read its list without a null test. **+2 on 2026-09-07 (P4e, §2.15 of `docs/plans/small-metal.md`)**: the reclamation fixes — one `unsafe` for the periodic `generic_collect` sweep and one for the reclaim-and-retry that runs before `malloc_generic` returns null. Both call `collect_inner`, which allocates nothing and touches only this heap’s own pages on the owner thread; each carries its SAFETY line. `malloc_generic` itself became a safe wrapper over the renamed `malloc_generic_once`, so the split added no signature. **+2 on 2026-09-08:** the medium collect-and-retry ahead of the heartbeat -- one `unsafe` around `page_collect` + `page_pop` on the bin queue front, one reading `free_is_zero` off the page just popped. Both are the SAME operations `malloc_generic_walk` performs on the same page a few lines later, on the owner thread under the heap lock: the block moves earlier, nothing new is dereferenced. Each carries its SAFETY line. **+2 on 2026-09-16 (Openheimer):** the two refuse arms in `malloc_aligned_at_slow` when `bins::aligned_at_from` reports that `block + offset + align` would wrap for a caller-chosen offset — each is one `free_local` of the block just allocated on this thread, which has not escaped. **+1 on 2026-09-24 (instruction-count campaign, LEDGER):** the medium collect-and-retry's `page_collect` + `page_pop` block became two blocks so the `saw_remote_free` latch is stored between them — same two calls on the same page, owner thread, in the same order; nothing new is dereferenced. **+1 on 2026-09-24 (round two):** a medium miss grows the front page directly — one block around the `capacity < reserved` test and the `grow_front` call, on the same page the block above just collected, which is `grow_front`'s whole contract **+2 on 2026-09-25 (CURIOSITY ROUND FOUR):** the medium arm pops the front page BEFORE collecting it — one `page_pop` block and the `free_is_zero` read beside it, the same two operations the collect-then-pop below already performs on the same owner-thread page | 2026-09-25 | +| `heap.rs` | 80 | Owner-thread page/queue manipulation under raw pointers (no two `&mut Page` may coexist), the aligned-allocation peek, span carving. **+3 on 2026-08-19**: `try_unlink_huge_segment` split out of `remove_huge_segment`. **+15 on 2026-08-22**: `malloc_generic` split into a small entry plus `malloc_generic_walk`, with `grow_front`, `try_guarded` and `drain_delayed` as cold arms — each split adds an `unsafe fn` signature and its block while dereferencing nothing the single function did not — and the immortal `EMPTY_DELAYED` sentinel that lets the heartbeat read its list without a null test. **+2 on 2026-09-07 (P4e, §2.15 of `docs/plans/small-metal.md`)**: the reclamation fixes — one `unsafe` for the periodic `generic_collect` sweep and one for the reclaim-and-retry that runs before `malloc_generic` returns null. Both call `collect_inner`, which allocates nothing and touches only this heap’s own pages on the owner thread; each carries its SAFETY line. `malloc_generic` itself became a safe wrapper over the renamed `malloc_generic_once`, so the split added no signature. **+2 on 2026-09-08:** the medium collect-and-retry ahead of the heartbeat -- one `unsafe` around `page_collect` + `page_pop` on the bin queue front, one reading `free_is_zero` off the page just popped. Both are the SAME operations `malloc_generic_walk` performs on the same page a few lines later, on the owner thread under the heap lock: the block moves earlier, nothing new is dereferenced. Each carries its SAFETY line. **+2 on 2026-09-16 (Openheimer):** the two refuse arms in `malloc_aligned_at_slow` when `bins::aligned_at_from` reports that `block + offset + align` would wrap for a caller-chosen offset — each is one `free_local` of the block just allocated on this thread, which has not escaped. **+1 on 2026-09-24 (instruction-count campaign, LEDGER):** the medium collect-and-retry's `page_collect` + `page_pop` block became two blocks so the `saw_remote_free` latch is stored between them — same two calls on the same page, owner thread, in the same order; nothing new is dereferenced. **+1 on 2026-09-24 (round two):** a medium miss grows the front page directly — one block around the `capacity < reserved` test and the `grow_front` call, on the same page the block above just collected, which is `grow_front`'s whole contract **+2 on 2026-09-25 (CURIOSITY ROUND FOUR):** the medium arm pops the front page BEFORE collecting it — one `page_pop` block and the `free_is_zero` read beside it, the same two operations the collect-then-pop below already performs on the same owner-thread page. **+1 on 2026-10-07 (`docs/opps.md` vein census, vein 2):** `collect_inner` hands the per-heap cached large span (`Heap::large_cache`) back with `retire_span` — one block around `segment_of` on the cached page's own address and the same `retire_span` a large free performed before the cache existed, on the owner thread. **+2 the same day:** `free_large_span` — an `unsafe fn` and its block — is the large-span arm of `free_local_at` moved into its own function (double-free check, cache, free-run flag); the same operations on the same page, owner thread | 2026-10-07 | | `init.rs` | 42 | Thread/heap lifecycle: the initial-exec TLS slot (`global_asm!` + fs-relative asm reads), thread-pointer register reads (`fs:0`/`gs:0x30`/`tpidrro_el0`), heap-box creation/teardown, the abandonment path run inside platform TLS destructors. **+1 on 2026-09-09 (`firmware-what-is-left.md` §7.3): an ATTRIBUTE, not an operation** — `unsafe(link_section = ".rodata.…")` on `EMPTY_HEAP_BOX`, bare metal only, so the never-written sentinel lives in flash instead of costing 1,752 bytes of RAM. The token is counted because the census is a text search; the contract it rests on (raw reads only, no page ever stores its address) is the one the `Sync` impl below already carries, and a write would now fault against the flash cache instead of silently landing. `create_heap`'s template copy from it is a `ptr::read` inside the block that already existed. **+5 on 2026-09-16 (Openheimer, OH-rusty_alloc-13):** thread exit now abandons EVERY heap the dying thread still owns, not only the one in `done_slot` — a first-class heap that was never `heap_delete`d, or one installed with `set_default_heap`, used to stay DELAYED under a dead owner forever. `take_heap_owned_by` unlinks one such box under `HEAPS_LOCK` (one block: the registry walk); `thread_done` reads `owner_tid` off its own box and calls the split-out `unsafe fn thread_done_one` for the primary box and for each extra one (three sites plus the signature). Nothing new is dereferenced that the single function did not; each carries its SAFETY line | 2026-09-16 | -| `segment.rs` | 43 | Segment/page metadata addressing: the mask trick (`segment_of`), `page_of`'s contract-based indexing — **the bound is now PROVED for every in-segment offset by `proofs.rs` (Kani), not merely asserted** — span tiling, purge/recommit. The baseline had drifted to 36 without a row edit; recorded now. **+2 on 2026-10-04 (`docs/plans/recommit-failure-ignored.md`):** `restore_for_reuse` — an `unsafe fn` and its block — is the guard-lift and re-commit `segment_free` and `huge_free` each did inline, moved into one function so both can act on its answer: a segment whose memory the OS will not make accessible and committed again is released whole or retired in place, never recycled. It touches the same range the inline code did; `segment_free`'s own block became the expression that calls it (no net change). `span_recommit` now returns whether the span is backed (same block, same call). **+5 the same day, all `#[cfg(test)]`:** `a_failed_recommit_is_a_failed_span_not_an_unbacked_one` writes and frees the blocks it allocated (four blocks) and frees them at the end (one); each carries its SAFETY line | 2026-10-04 | +| `segment.rs` | 45 | Segment/page metadata addressing: the mask trick (`segment_of`), `page_of`'s contract-based indexing — **the bound is now PROVED for every in-segment offset by `proofs.rs` (Kani), not merely asserted** — span tiling, purge/recommit. The baseline had drifted to 36 without a row edit; recorded now. **+2 on 2026-10-04 (`docs/plans/recommit-failure-ignored.md`):** `restore_for_reuse` — an `unsafe fn` and its block — is the guard-lift and re-commit `segment_free` and `huge_free` each did inline, moved into one function so both can act on its answer: a segment whose memory the OS will not make accessible and committed again is released whole or retired in place, never recycled. It touches the same range the inline code did; `segment_free`'s own block became the expression that calls it (no net change). `span_recommit` now returns whether the span is backed (same block, same call). **+5 the same day, all `#[cfg(test)]`:** `a_failed_recommit_is_a_failed_span_not_an_unbacked_one` writes and frees the blocks it allocated (four blocks) and frees them at the end (one); each carries its SAFETY line. **+2 on 2026-10-07 (`docs/opps.md` vein census, vein 1):** `scrub_recycled` — an `unsafe fn` and its block — replaces the whole-header `write_bytes` that `segment_alloc` and `huge_alloc` did on every recycled chunk: it zeroes the 2 KB owner table (all of it for a Normal segment, entry 0 for a Huge one, which overwrites the rest) and the first `slots` page slots, inside the same header the old call covered. The other slots of a Normal segment are zeroed where the bump cursor first carves them (inside `span_alloc`'s existing block); a Huge segment reads only slots 0 and 1, and `wait_no_remote_in_flight` now scans slot 1 alone there. Nothing new is dereferenced | 2026-10-07 | | `prim/windows.rs` | 34 | OS FFI: VirtualAlloc family, FLS destructors, QPC, BCryptGenRandom. **+3 on 2026-09-16 (Openheimer):** `range_is_reserved` (`zeroed` `MEMORY_BASIC_INFORMATION` + `VirtualQuery`, the same OS answer `unix.rs` gets from `mincore`), and `alloc_aligned` now refuses a garbage alignment and a `size + align` that wraps BEFORE it reserves anything, and releases a re-reservation that the OS placed anywhere but the aligned address it asked for instead of returning it misaligned (`VirtualFree`, one block). **+1 on 2026-09-24 (`docs/plans/youslowbro.md` §4):** `getenv` — one `GetEnvironmentVariableA` into a caller buffer of exactly the length passed, the allocation-free environment read that replaced `std::env::var`'s owned strings in the options pass (251 allocations through the global allocator on every process's first allocation, now none) | 2026-09-24 | | `page.rs` | 38 | Free-list links written into dead blocks, the lock-free `xthread_free` four-state protocol (loom-modeled in `tests/loom_xthread.rs`), the immortal `EMPTY_PAGE` sentinel — **+1 on 2026-09-09: the same `unsafe(link_section)` attribute as `init.rs`, placing it in flash on bare metal; its free list is permanently null, so nothing writes it**. **+8 on 2026-08-22**: `page_link_local` split out of `page_push_local` so the caller can decrement `used` in one memory-destination RMW, `page_collect_impl` const-generic over whether it also writes the protocol flag, and `USED_OFFSET` — whose value is asserted against `offset_of!(Page, used)` by a unit test, because an asm operand is not type-checked and a field reordering would silently decrement the wrong bytes | 2026-08-22 (free campaign) | | `prim/unix.rs` | 33 | OS FFI: mmap family, madvise/decommit, pthread keys, /dev/urandom. **+1 on 2026-09-16 (Openheimer, OH-201):** `range_is_reserved` asks `mincore` whether a caller-supplied `manage_os_memory` range is mapped at all before it becomes an arena — one FFI call on a page-aligned probe. **+1 on 2026-09-24 (`docs/plans/youslowbro.md` §4):** `getenv` — `libc::getenv` and a byte-by-byte read of the returned C string up to its NUL or the caller buffer's end, nothing written through it; the same call upstream's prim makes, with the same standing caveat about a concurrent `setenv` **+2 on 2026-09-25:** `env_for_each` — an `unsafe extern "C"` declaration of `environ` and one block that walks it to its NULL terminator, handing each NUL-terminated entry on as a pointer; read only, with the same concurrent-`setenv` caveat as `getenv`. **+2 on 2026-10-01 (macOS):** the Apple arm of `range_is_reserved` — an `unsafe extern "C"` declaration of `mach_vm_region` and the `mach_task_self_` global, and one call per region: a read-only query of our own task into a `packed(4)` `vm_region_basic_info_64` whose 36-byte size is asserted at compile time and is exactly the word count passed, because XNU's `mincore` succeeds on unmapped ranges | 2026-10-01 | @@ -31,7 +31,7 @@ unsafe in DEPENDENCIES, and ours are `libc` plus bindings-only `windows-sys`. | `rusty_alloc_api/src/lib.rs` | 16 | The `GlobalAlloc`/`Allocator` impls forwarding layouts to the core; `unsafe` is inherent to those traits' contracts . **+1 on 2026-09-24:** `GlobalAlloc::alloc`'s over-aligned arm calls `alloc::malloc_aligned_pow2`, an `unsafe fn` whose one added precondition — a power-of-two alignment — `Layout` guarantees **+1 on 2026-09-25:** the over-aligned `realloc` arm reads `usable_size(ptr)` to keep a block in place — `ptr` is a live non-null block of ours by the `GlobalAlloc` contract | 2026-09-25 | | `os.rs` | 12 | The prim-layer wrapper: commit/decommit/protect plumbing | 2026-08-08 | | `prim/mock.rs` | 8 | Miri-only mock OS backend (never shipped; `cfg(miri)`) | 2026-08-06 | -| `arena.rs` | 16 | Lock-free chunk bitmap claim/verify, recycled-chunk scrubbing (the 0.1.0-alpha.2 UAF fix lives here: `wait_no_remote_in_flight` on every recycle path). The baseline had drifted to 13 without a row edit; recorded now. **+2 on 2026-09-16 (Openheimer, `docs/plans/openheimer-run.md`):** `arena_register` claims its slot with a CAS instead of a load-then-store, and the two new blocks are the REFUSE arm — `os::free` of the descriptor and of the OS mapping when the table is full or the slot was lost — so a refused arena leaks nothing. Both free memory this function allocated a few lines earlier and nothing else has seen; each carries its SAFETY line. **+1 on 2026-10-04 (`docs/plans/recommit-failure-ignored.md`):** `owns` — the range scan `chunk_free` already performs, asked without clearing the used bit, so the release path can tell an arena chunk (retire in place) from a reservation of its own (release to the OS). It reads each live descriptor's `base` and `chunks_live` and nothing else | 2026-10-04 | +| `arena.rs` | 15 | Lock-free chunk bitmap claim/verify, recycled-chunk scrubbing (the 0.1.0-alpha.2 UAF fix lives here: `wait_no_remote_in_flight` on every recycle path). The baseline had drifted to 13 without a row edit; recorded now. **+2 on 2026-09-16 (Openheimer, `docs/plans/openheimer-run.md`):** `arena_register` claims its slot with a CAS instead of a load-then-store, and the two new blocks are the REFUSE arm — `os::free` of the descriptor and of the OS mapping when the table is full or the slot was lost — so a refused arena leaks nothing. Both free memory this function allocated a few lines earlier and nothing else has seen; each carries its SAFETY line. **+1 on 2026-10-04 (`docs/plans/recommit-failure-ignored.md`):** `owns` — the range scan `chunk_free` already performs, asked without clearing the used bit, so the release path can tell an arena chunk (retire in place) from a reservation of its own (release to the OS). It reads each live descriptor's `base` and `chunks_live` and nothing else. **-1 on 2026-10-07:** `owns`, `chunk_free` and `chunk_free_n` now share one table scan, `find_arena` (`chunk_free` is `chunk_free_n(p, 1)`), so the three copies of the scan block became one | 2026-10-07 | | `prim/fixed.rs` | 37 | **New 2026-09-07 (P1 of `docs/plans/small-metal.md`).** The fixed-region backend for a target with no OS: memory is a `&'static mut [u8]` handed over once. **6 of the 18 are `unsafe fn` signatures the prim seam requires** (`alloc`/`free`/`commit`/`decommit`/`reset`/`protect`) whose *bodies contain no unsafe operation at all* — the free list is `AtomicUsize` arrays under a spin lock and the pointers are built with the safe `with_exposed_provenance_mut`, so this backend adds **zero** unsafe dereferences to the shipped crate. The other 12 are in `#[cfg(test)]`: two `&raw mut` static-region handoffs and ten calls through the `unsafe fn` seam, each with its SAFETY line. **+1 on 2026-09-07 (P2):** the region test became two-sided — where the shipped geometry refuses a segment-sized request from a 512 KiB region, the small profile SERVES one, so the test now frees it too. Audited at the site; the module is `allow(dead_code)` and unreachable on every platform that has an arm above it. **+7 on 2026-09-07 (P4b, §2.9/§2.10 of the same plan):** the two-ended `place` rule added ZERO unsafe to shipped code — `place` is a pure arithmetic `fn` and the scan around it is unchanged — and all seven are in `#[cfg(test)]`: one `slice::from_raw_parts_mut` carving the `REGION_ALIGN`-aligned window out of `BACKING` (replacing a `&mut *ptr` that a `repr(align(65536))` static would have needed, which rustc 1.97.1 on MSVC cannot compile), one `ptr::add` to reach that window, and five calls through the `unsafe fn` seam in the placement assertions and in `greedy_segments`, which allocates segments until refusal and frees every one before returning. Each carries its SAFETY line **+1 on 2026-09-09 (morning, `#21`):** the two-sided alignment test frees the SEGMENT_SIZE-aligned page it is served when the region straddles a boundary — `#[cfg(test)]`, banked without a row here at the time; recorded now. **+5 on 2026-09-09 (`docs/plans/finished/region-alignment-bug.md` §7): the module ships its first unsafe OPERATIONS.** (1) `unsafe impl Sync for Region` and (2) the `&mut *self.bytes.get()` in `Region::give` — the once-only handoff the Janus seam has carried since 2.0.0, moved here so the alignment travels with it; a module-wide `REGION_GIVEN` swap precedes the `&mut`, so a second `give` on any instance is refused before it could alias. (3) `unsafe impl Sync for FirstHeapBox`, the bare-metal static holding the first heap's descriptor (handed out once by `take_first_heap_box`, on the one thread such a build has). The other two are `#[cfg(test)]`: a `from_raw_parts_mut` carving the report's misaligned base out of a static for the refusal probe, and a `Box::new_zeroed().assume_init()` allocating a `Region` in place, because materialising a 64 KiB-aligned value on the stack first faults on Windows. Each carries its SAFETY line. **+2 on 2026-09-09 (`docs/plans/finished/region-alignment-dissolve.md`): zero new unsafe in shipped code** — segments now stride from the region's base, and every piece of that is safe: `stride_base` is a relaxed load, `place` takes an `origin` and does arithmetic, `install_region` aligns the base up. The two are `#[cfg(test)]`, the `--cfg ra_aligned_region` arm of the two-sided alignment test (an `alloc` and a `free` through the `unsafe fn` seam: the 2.0.4 straddle case, kept under the knob that restores the mask), and the `Box::new_zeroed().assume_init()` moved under that same cfg — the default arm is a plain `static Region` now, which a 16-byte-aligned type can be on every host toolchain. Each carries its SAFETY line. **+4 on 2026-09-09 (`docs/plans/finished/esp32-large-alloc-ceiling.md`): all four are `#[cfg(test)]`, and shipped code gained none.** The large-allocation ceiling reported from `rusty_zstd` is reproduced against the real extent allocator by `greedy_dedicated`, which serves `huge_alloc`-shaped reservations until the region refuses and frees every one before returning (two calls through the `unsafe fn` seam), plus one `alloc`/`free` pair modelling the whole segment a first small allocation claims. The fix itself -- the `ra_segment_size` geometry knob, `dedicated_segments`, `region_for_allocs` and `region_capacity` -- is arithmetic over the existing atomics and adds no unsafe operation at all. Each carries its SAFETY line | 2026-09-09 | | `prim/wasm.rs` | 6 | `memory.grow` linear-memory backend | 2026-08-06 | | `options.rs` | 17 | Env parsing at init, registered-hook invocation. **+1 on 2026-09-24: `#[cfg(test)]` only** — `std::env::set_var` (an `unsafe fn` in edition 2024) seeding two probe variables for the in-place lookup test. Shipped code gained none: the pass itself went from 251 allocations to zero by building its keys in a stack buffer and reading through `prim::getenv` **+10 on 2026-09-25:** the one-walk environment pass (Linux) — three `unsafe fn` helpers (`strip`, `match_name`, `copy_value`) that read a NUL-terminated entry byte by byte and stop at the first mismatch or its NUL, and the blocks that call them; nothing is written through an entry | 2026-09-25 | diff --git a/crates/rusty_alloc/src/arena.rs b/crates/rusty_alloc/src/arena.rs index 3300cd7..87d2a46 100644 --- a/crates/rusty_alloc/src/arena.rs +++ b/crates/rusty_alloc/src/arena.rs @@ -435,12 +435,55 @@ fn chunk_alloc_n_inner(restrict_id: i32, n: usize) -> Option<(*mut u8, bool)> { } let mut run = 0usize; let mut idx = 0usize; + // The bitmap word under the cursor, loaded once per word, not + // once per bit: the scan was an atomic load per chunk, ~166 Ir + // per huge allocation (`docs/opps.md` vein census). A cached + // word only steers the scan; every claim below is verified + // against the live word, so a stale copy cannot double-assign. + let mut cur_w = usize::MAX; + let mut used_w: BitWord = 0; while idx < chunks { - let bit = (*a).used[idx / WORD_BITS].load(Ordering::Acquire) - & (1 << (idx % WORD_BITS)); + let w = idx / WORD_BITS; + if w != cur_w { + cur_w = w; + used_w = (*a).used[w].load(Ordering::Acquire); + if used_w == BitWord::MAX { + // Nothing free in this word: the run ends here. + run = 0; + idx = (w + 1) * WORD_BITS; + continue; + } + } + let bit = used_w & (1 << (idx % WORD_BITS)); run = if bit == 0 { run + 1 } else { 0 }; if run == n { let start = idx + 1 - n; + // A run inside ONE word is claimed with one `fetch_or` + // of its mask (and its dirty bits with another), not + // one locked RMW per chunk for each. Same contract as + // the per-chunk path below: on conflict, roll back + // exactly the bits this claim set and rescan past the + // lowest conflicting chunk. + if start / WORD_BITS == w { + let lo = start % WORD_BITS; + let ones: BitWord = if n >= WORD_BITS { + BitWord::MAX + } else { + ((1 as BitWord) << n) - 1 + }; + let mask = ones << lo; + let prev = (*a).used[w].fetch_or(mask, Ordering::AcqRel); + if prev & mask != 0 { + (*a).used[w].fetch_and(!(mask & !prev), Ordering::AcqRel); + run = 0; + idx = w * WORD_BITS + (prev & mask).trailing_zeros() as usize + 1; + cur_w = usize::MAX; // reload the live word + continue; + } + let dirty_prev = (*a).dirty[w].fetch_or(mask, Ordering::AcqRel); + let p = (*a).base.add(start * SEGMENT_SIZE); + return Some((p, dirty_prev & mask == 0)); + } // CLAIM-AND-VERIFY. MULTI_LOCK excludes other // multi-chunk claims but NOT the lock-free // single-chunk path, which can take one of these @@ -465,6 +508,7 @@ fn chunk_alloc_n_inner(restrict_id: i32, n: usize) -> Option<(*mut u8, bool)> { } run = 0; idx = c + 1; + cur_w = usize::MAX; // reload the live word continue; } let mut any_dirty = false; @@ -487,25 +531,21 @@ fn chunk_alloc_n_inner(restrict_id: i32, n: usize) -> Option<(*mut u8, bool)> { result } -/// Free `n` contiguous chunks. True when the address belonged to an arena. +/// The live arena whose chunks contain `addr`, with `addr`'s byte offset into +/// it and its live chunk count. The one copy of the table scan that +/// [`chunk_free_n`], [`chunk_free`] and [`owns`] each used to carry. #[allow(clippy::needless_range_loop)] // indexed scan over a fixed atomic table -pub fn chunk_free_n(p: *mut u8, n: usize) -> bool { +fn find_arena(addr: usize) -> Option<(*mut Arena, usize, usize)> { if crate::FIXED_REGION { - return false; // nothing came from an arena, so nothing returns to one - } - // Zero / overflowing `n` is false, not `start..start+n` OOB on the bitmap - // (OH-rusty_alloc-25). - if n == 0 { - return false; + return None; // nothing came from an arena, so nothing returns to one } - let addr = p.addr(); let count = ARENA_COUNT.load(Ordering::Acquire).min(MAX_ARENAS); for id in 0..count { let a = ARENAS[id].load(Ordering::Acquire); if a.is_null() { continue; } - // SAFETY: live descriptor; indices clamped to live chunks. + // SAFETY: live descriptor; only its base and live-chunk count are read. unsafe { let chunks = (*a).chunks_live.load(Ordering::Acquire); let Some(span) = chunks.checked_mul(SEGMENT_SIZE) else { @@ -514,29 +554,43 @@ pub fn chunk_free_n(p: *mut u8, n: usize) -> bool { let Some(end_addr) = (*a).base.addr().checked_add(span) else { continue; }; - if addr < (*a).base.addr() || addr >= end_addr { - continue; - } - let off = addr - (*a).base.addr(); - // Interior pointers are false, not a successful free of the - // containing chunk (OH-rusty_alloc-27). - if !off.is_multiple_of(SEGMENT_SIZE) { - return false; - } - let start = off / SEGMENT_SIZE; - let Some(end) = start.checked_add(n) else { - return false; - }; - if end > chunks { - return false; - } - for j in start..end { - (*a).used[j / WORD_BITS].fetch_and(!(1 << (j % WORD_BITS)), Ordering::AcqRel); + if addr >= (*a).base.addr() && addr < end_addr { + return Some((a, addr - (*a).base.addr(), chunks)); } - return true; } } - false + None +} + +/// Free `n` contiguous chunks. True when the address belonged to an arena. +pub fn chunk_free_n(p: *mut u8, n: usize) -> bool { + // Zero / overflowing `n` is false, not `start..start+n` OOB on the bitmap + // (OH-rusty_alloc-25). + if n == 0 { + return false; + } + let Some((a, off, chunks)) = find_arena(p.addr()) else { + return false; + }; + // Interior pointers are false, not a successful free of the containing + // chunk (OH-rusty_alloc-27). + if !off.is_multiple_of(SEGMENT_SIZE) { + return false; + } + let start = off / SEGMENT_SIZE; + let Some(end) = start.checked_add(n) else { + return false; + }; + if end > chunks { + return false; + } + // SAFETY: live descriptor; `start..end` lies inside its live chunks. + unsafe { + for j in start..end { + (*a).used[j / WORD_BITS].fetch_and(!(1 << (j % WORD_BITS)), Ordering::AcqRel); + } + } + true } /// Whether `p` lies inside any arena's live chunks — the question @@ -547,71 +601,13 @@ pub fn chunk_free_n(p: *mut u8, n: usize) -> bool { /// out as-is), and a chunk inside an arena's reservation cannot be released /// to the OS on its own either. Such a segment is retired in place, and this /// is how the release path tells the two cases apart. -#[allow(clippy::needless_range_loop)] // indexed scan over a fixed atomic table pub fn owns(p: *const u8) -> bool { - if crate::FIXED_REGION { - return false; - } - let addr = p.addr(); - let n = ARENA_COUNT.load(Ordering::Acquire).min(MAX_ARENAS); - for id in 0..n { - let a = ARENAS[id].load(Ordering::Acquire); - if a.is_null() { - continue; - } - // SAFETY: live descriptor; only its base and live-chunk count are read. - unsafe { - let chunks = (*a).chunks_live.load(Ordering::Acquire); - let Some(span) = chunks.checked_mul(SEGMENT_SIZE) else { - continue; - }; - let Some(end_addr) = (*a).base.addr().checked_add(span) else { - continue; - }; - if addr >= (*a).base.addr() && addr < end_addr { - return true; - } - } - } - false + find_arena(p.addr()).is_some() } /// Return a chunk to its arena. True when the address belonged to one. -#[allow(clippy::needless_range_loop)] // indexed scan over a fixed atomic table pub fn chunk_free(p: *mut u8) -> bool { - if crate::FIXED_REGION { - return false; - } - let addr = p.addr(); - let n = ARENA_COUNT.load(Ordering::Acquire).min(MAX_ARENAS); - for id in 0..n { - let a = ARENAS[id].load(Ordering::Acquire); - if a.is_null() { - continue; - } - // SAFETY: live descriptor; bit index bounded by the range check. - unsafe { - let chunks = (*a).chunks_live.load(Ordering::Acquire); - let Some(span) = chunks.checked_mul(SEGMENT_SIZE) else { - continue; - }; - let Some(end_addr) = (*a).base.addr().checked_add(span) else { - continue; - }; - if addr >= (*a).base.addr() && addr < end_addr { - let off = addr - (*a).base.addr(); - // Interior pointers are false, not a successful free of the - // containing chunk (OH-rusty_alloc-27). - if !off.is_multiple_of(SEGMENT_SIZE) { - return false; - } - let idx = off / SEGMENT_SIZE; - (*a).used[idx / WORD_BITS].fetch_and(!(1 << (idx % WORD_BITS)), Ordering::AcqRel); - return true; - } - } - } - false + chunk_free_n(p, 1) } /// Adopt an OS block that cannot be returned to the host, making its memory diff --git a/crates/rusty_alloc/src/heap.rs b/crates/rusty_alloc/src/heap.rs index f3c5f62..1a8a0a8 100644 --- a/crates/rusty_alloc/src/heap.rs +++ b/crates/rusty_alloc/src/heap.rs @@ -149,6 +149,11 @@ impl Default for Heap { } } +/// The largest span [`Heap::large_cache`] keeps: 4 MiB. Bigger spans are +/// rarer, and a cached one is space no other size can use until it is +/// flushed. +pub const LARGE_CACHE_MAX_SLICES: usize = (4 << 20) / SEGMENT_SLICE_SIZE; + /// The heap. pub struct Heap { /// Page queues, indexed by bin; `pages[BIN_FULL]` parks full pages. @@ -168,6 +173,24 @@ pub struct Heap { /// Huge (dedicated-segment) allocations owned by this heap, so /// delete/destroy/abandon can account for them. pub huge_segments: *mut Segment, + /// One freed large span (single-block, unqueued) kept carved for the next + /// large allocation of the SAME slice count, or null. A large block is + /// otherwise carved fresh on every allocation and retired on every free: + /// a 2 MiB malloc/free pair cost 293 Ir, against 92 for the queued sizes + /// below it, and this cache takes it to ~124 (`docs/opps.md`, vein + /// census 2026-10-07, vein 2). Cached only with purging off, where a + /// freed span stays resident anyway, and only up to + /// [`LARGE_CACHE_MAX_SLICES`], so it cannot hold back much space other + /// sizes could have used; flushed by every collect, which includes the + /// heartbeat and thread teardown. A cached page has `used == 0`, which is + /// also what makes a second free of its block abort. + pub large_cache: *mut Page, + /// A large free has happened since the last large allocation: the cache + /// keeps only a LONE free, never one at the end of a teardown run. A + /// `bool`, not a counter: a `u32` here changed `Heap`'s layout and cost + /// every allocation path (big +4.00, mixed +2.89 Ir/op) though none of + /// them reads it; the flag packs into existing padding. + pub large_freed: bool, /// Arena restriction: allocate segments only from this arena (−1 = any /// arena, then the OS — `mi_heap_new_in_arena`). pub arena_id: i32, @@ -238,6 +261,8 @@ impl Heap { empty_segments: 0, delayed: &raw const EMPTY_DELAYED, huge_segments: ptr::null_mut(), + large_cache: ptr::null_mut(), + large_freed: false, arena_id: -1, tag: 0, rng: crate::random::Random::new(), @@ -1050,6 +1075,26 @@ impl Heap { let slices = size.div_ceil(SEGMENT_SLICE_SIZE); // SAFETY: heap lock held; page/segment owned by this heap. unsafe { + // Same-size reuse of the span the last large free kept. Its page + // is still fully built (bin, flags, `xheap`, `block_size`); only + // the liveness and the contents' zero-ness change. + self.large_freed = false; + let c = self.large_cache; + if !c.is_null() && (*c).slice_count as usize == slices { + self.large_cache = ptr::null_mut(); + (*c).used = 1; + (*c).free_is_zero = false; + (*c).heap_tag = self.tag; + self.stat_alloc(); + self.stats.large_allocs += 1; + return ((*c).area, false); + } + // REFUTED (2026-10-07): retiring the cached span here on a size + // miss, so it could coalesce before this allocation carved. It + // did not fix the model-swap spill (the free-run rule in + // `free_large_span` did), and the extra call, inlined into + // `malloc_generic_once`, cost every generic allocation +3 Ir (big + // 92 -> 95, mixed +2.2) though none of them is large. let Some((p, fresh)) = self.span_from_segments(slices) else { return (ptr::null_mut(), false); }; @@ -1500,6 +1545,19 @@ impl Heap { /// # Safety /// Owner thread; `reclaim` only when this heap will REMAIN live. unsafe fn collect_inner(&mut self, _force: bool, reclaim: bool) { + // Hand the cached large span back first, so a collect leaves no span + // carved that holds no block (see `large_cache`). + let c = self.large_cache; + if !c.is_null() { + self.large_cache = ptr::null_mut(); + // SAFETY: a cached page is an unqueued, blockless span start of + // one of this heap's Normal segments (the page slot lives in its + // header, so the mask finds the segment). + unsafe { + let seg = segment_of(c.cast::()); + let _ = self.retire_span(seg, c); // `seg` unused after + } + } // SAFETY: owner thread per contract. unsafe { // `force` RECLAIMS ABANDONED SEGMENTS. It used to be ignored @@ -1709,7 +1767,20 @@ impl Heap { } // Adopted large spans that are already dead: retire them now that // the walk is over (the layout is ours to mutate again). + // + // ALL of them, restarting the walk after each retire (it coalesces, + // so `idx` may now sit inside a merged free span). This loop used + // to retire ONE and `break` with "leave the rest to later + // collects" — but a large span is never queued, so no collect ever + // visits it again: every other dead span stayed carved for the + // life of the process. That is how a model loaded on one thread, + // which then exited, and dropped on another stranded ~0.6 of the + // model's size whatever `purge_delay` said + // (`docs/plans/huge-free-retention.md`, mechanism 1). Upstream + // collects every concurrent free of an abandoned segment + // (`mi_segment_check_free` in `mi_segment_try_reclaim`). let mut idx = segment::HEADER_SLICES; + let mut retired_any = false; while saw_large_span && idx < (*seg).next_free_slice as usize { let slot: *mut Page = &raw mut (*seg).pages[idx]; let len = ((*slot).slice_count as usize).max(1); @@ -1725,11 +1796,29 @@ impl Heap { if !self.retire_span(seg, slot) { return false; // RELEASED during adoption } - break; // layout changed: leave the rest to later collects + retired_any = true; + // The retire coalesced: `idx` may now be inside the merged + // free span. Every slot of a span points back to its start + // (`slice_offset`, the invariant `debug_validate_segment` + // checks), so resume at that start and step over the merged + // span. Restarting from slice 0 instead is quadratic in the + // span count; measured flat at 64 spans (139,722 vs 139,729 + // Ir per op), so this bounds the worst case and is not a + // win. The +2.1 % that adoption costs over 2.2.3 there is + // retiring the other 63 dead spans, which 2.2.3 stranded. + idx -= (*slot).slice_offset as usize; + continue; } idx += len; } - if (*seg).used_pages == 0 { + // A segment that ARRIVED empty is accounted here. One emptied by + // the retires above was already accounted by `retire_span` (it + // parked it as this heap's empty segment, or released it and + // returned false); counting it again released the very segment + // just parked, leaving `empty_segments == 1` with no empty segment + // behind it, after which this heap released every empty segment + // instead of keeping one. + if !retired_any && (*seg).used_pages == 0 { if self.empty_segments == 0 { self.empty_segments = 1; } else if self.remove_segment(seg) { @@ -1878,7 +1967,13 @@ impl Heap { SegmentKind::Normal => { self.stat_free(); if (*pg).bin as usize == BIN_HUGE { - let _ = self.retire_span(seg, pg); // returns below + // Its own function for readability; `#[inline]`. It was + // `#[inline(never)]` while a +3 Ir regression on every + // generic op was blamed on this function: bisected, the + // regression was a flush-on-miss in `large_alloc`, and + // inlining this back measured 2 MiB pair 147 -> 138 with + // every other op unchanged. + self.free_large_span(seg, pg); return; } let used_now = page_push_local(pg, p.cast()); @@ -1925,6 +2020,57 @@ impl Heap { } } + /// Free the one block of a large (single-block, unqueued) span: the + /// double-free check, then the per-heap large cache (`large_cache`). + /// + /// # Safety + /// Heap lock held; `pg` is a large span start of `seg`, owned by this + /// heap, whose block is being freed. + #[inline] + unsafe fn free_large_span(&mut self, seg: *mut Segment, pg: *mut Page) { + // SAFETY: forwarded contract. + unsafe { + // One block per large span: `used` is 1 while it + // lives and 0 once freed (and possibly cached), so a + // second free of it is a double free. + if (*pg).used == 0 { + crate::page::double_free_abort(); + } + // Retire whatever is cached FIRST, so spans retire in + // free order and coalesce exactly as without a cache. + // (Retiring the cached one last cost +882 Ir per op on + // the exited-thread reload probe: later frees had + // coalesced into a run to its right, and absorbing it + // re-marked every slice.) `pg` stays carved, so its + // segment cannot empty here; `old`'s may. + let old = core::mem::replace(&mut self.large_cache, ptr::null_mut()); + if !old.is_null() { + let _ = self.retire_span(segment_of(old.cast::()), old); // terminal: `old` and its segment are not touched after + } + // Cache only a LONE free: the first since the last + // large allocation, which is the alloc/free loop the + // cache serves. A run of large frees is a teardown + // (dropping a model, a batch, a request's buffers), + // and a span cached at its end is refilled out of + // address order by the next load, which then spills + // into a fresh segment: a same-thread model swap + // peaked at 1.04 models instead of 1.01 until this + // (2026-10-07 validation sweep). Flushing on a size + // miss alone did not fix it. + let lone = !self.large_freed; + self.large_freed = true; + if lone + && (*pg).slice_count as usize <= LARGE_CACHE_MAX_SLICES + && crate::options::get(15) < 0 + { + (*pg).used = 0; + self.large_cache = pg; + return; + } + let _ = self.retire_span(seg, pg); // terminal: `seg` is not touched after + } + } + /// Return a span to its segment; release the segment when it empties. /// /// **Returns `false` when the SEGMENT was released** — `seg` is unmapped and diff --git a/crates/rusty_alloc/src/segment.rs b/crates/rusty_alloc/src/segment.rs index 5607c85..fecdd5b 100644 --- a/crates/rusty_alloc/src/segment.rs +++ b/crates/rusty_alloc/src/segment.rs @@ -246,8 +246,10 @@ pub unsafe fn page_of(seg: *mut Segment, p: *mut u8) -> *mut Page { let off = *tab.add(idx) as usize; // The two representations must agree. This is the check that makes the // duplicated state safe to keep, and it is free in release. + // (A Huge segment keeps `page_off` alone: its interior slots' offsets + // are not written, see `huge_alloc`.) debug_assert!( - { + (*seg).kind == SegmentKind::Huge || { let base: *mut Page = (&raw mut (*seg).pages).cast(); let slot = base.add(idx); let back = (*slot).slice_offset as usize * slot_stride(); @@ -308,8 +310,10 @@ pub unsafe fn page_index(seg: *mut Segment, page: *mut Page) -> usize { // SAFETY: both point into the same header per the contract. unsafe { let idx = ((*page).area.addr() - seg.addr()) / SEGMENT_SLICE_SIZE; + // (A Huge segment keeps `page_off` alone: its interior slots' offsets + // are not written, see `huge_alloc`.) debug_assert!( - { + (*seg).kind == SegmentKind::Huge || { let base: *mut Page = (&raw mut (*seg).pages).cast(); idx == (page.addr() - base.addr()) / core::mem::size_of::() }, @@ -352,7 +356,17 @@ unsafe fn wait_no_remote_in_flight(seg: *mut Segment) { // that carved 10 slices scans 10, not 512. (A Huge segment sets // `next_free_slice = SLICES_PER_SEGMENT`, so it is unaffected — correct, // since its one page occupies the whole reservation.) - let end = (*seg).next_free_slice as usize; + // + // A Huge segment's one page is slot 1, so slot 1 is the only one + // scanned there. Its other slots are never read by anything but this + // scan (every slice resolves to slot 1 through `page_off`), and a + // recycled chunk no longer scrubs them (`scrub_recycled`): sweeping + // them read stale bytes, and cost 511 atomic loads per huge free. + let end = if (*seg).kind == SegmentKind::Huge { + HEADER_SLICES + 1 + } else { + (*seg).next_free_slice as usize + }; for i in HEADER_SLICES..end { let pg = base.add(i); while (*pg) @@ -367,6 +381,35 @@ unsafe fn wait_no_remote_in_flight(seg: *mut Segment) { } } +/// Scrub what the next tenant of a RECYCLED chunk could read before it writes +/// it: the owner table (so an uncarved slice resolves exactly as on fresh +/// memory) and the first `slots` page slots. The rest of `pages` is left as +/// it was. +/// +/// This used to zero the whole `Segment`, 512 slots of 88 bytes: 45 KB +/// written, and its pages first-touched, on every thread's first segment and +/// every huge allocation once the arena had warmed up. That was 47 k of the +/// 53.5 k instructions of a 64 MiB malloc/free pair and almost all of what a +/// thread's life cost over glibc (`docs/opps.md`, vein census 2026-10-07). A +/// Normal segment's slots are zeroed instead as the bump cursor first carves +/// them; a Huge segment reads only slots 0 and 1. +/// +/// # Safety +/// `seg` is the base of a mapping at least `size_of::()` long that +/// no one else is using. +/// +/// `owner_table` is false for a Huge segment, which overwrites every entry of +/// the owner table but the first a few lines later: zeroing it first was a +/// 2 KB memset that bought nothing. +unsafe fn scrub_recycled(seg: *mut Segment, slots: usize, owner_table: bool) { + // SAFETY: per contract, both ranges lie inside the header. + unsafe { + let tab = (&raw mut (*seg).page_off).cast::(); + core::ptr::write_bytes(tab, 0, if owner_table { SLICES_PER_SEGMENT } else { 1 }); + core::ptr::write_bytes((&raw mut (*seg).pages).cast::(), 0, slots); + } +} + /// Reserve a Normal segment — from an arena when one qualifies (respecting a /// heap's `arena_id` restriction), else eagerly-committed fresh OS memory — /// and register it in the segment map. @@ -386,9 +429,12 @@ pub fn segment_alloc(arena_id: i32) -> Result<*mut Segment, PrimError> { // in full. A recycled chunk's stale bytes are all overwritten here and // page slots are re-initialized by span_mark on every carve. unsafe { - // Recycled chunks carry stale page slots — scrub the header region. + // Recycled chunks carry stale page slots. Only the owner table and the + // header slot are scrubbed here; each later slot is zeroed when the + // bump cursor first carves it (`span_alloc`). Every scalar field is + // assigned below. if !mem_zero { - core::ptr::write_bytes(seg.cast::(), 0, core::mem::size_of::()); + scrub_recycled(seg, HEADER_SLICES, true); } (*seg).kind = SegmentKind::Normal; (*seg).total_size = size; @@ -720,6 +766,13 @@ pub unsafe fn span_alloc(seg: *mut Segment, slices: usize) -> (*mut Page, bool) if idx + slices > SLICES_PER_SEGMENT { return (ptr::null_mut(), false); } + // First carve of these slots in this tenancy: on a recycled chunk + // they still hold the previous tenant's bytes (`scrub_recycled` left + // them), and a stale `free_is_zero` or `xthread_free` would be read + // as this page's. On fresh OS memory they are already zero. + if !(*seg).mem_is_zero { + ptr::write_bytes((&raw mut (*seg).pages).cast::().add(idx), 0, slices); + } (*seg).next_free_slice = (idx + slices) as u32; (*seg).used_pages += 1; let start: *mut Page = &raw mut (*seg).pages[idx]; @@ -963,10 +1016,14 @@ pub fn huge_alloc( } }; let seg: *mut Segment = bptr.cast(); - // SAFETY: header region fully written below; recycled chunks scrubbed. + // SAFETY: header scalars fully written below; a recycled chunk's owner + // table and the two slots a huge segment uses (0, and 1 for its page) are + // scrubbed. Slots 2.. only get `slice_offset` below and are read by + // nothing else (every slice resolves to slot 1 through `page_off`, and + // `wait_no_remote_in_flight` scans slot 1 alone for a Huge segment). unsafe { if !mem_zero { - core::ptr::write_bytes(seg.cast::(), 0, core::mem::size_of::()); + scrub_recycled(seg, HEADER_SLICES + 1, false); } } let b = os::OsBlock { @@ -1048,18 +1105,17 @@ pub fn huge_alloc( ); (*page).free_is_zero = b.is_zero; // Every slice of a huge segment resolves to slot 1, so the owner table - // is one value repeated across the run. (Writing it as a vectorised - // `slice::fill` instead of a store in the loop below measured exactly - // flat — the loop is dominated by `slice_offset`, which strides by - // `size_of::()` and cannot vectorise.) + // is one value repeated across the run, and it is the ONLY record of + // that: the interior slots' `slice_offset` used to be written too, 511 + // stores 88 bytes apart that touched the whole 45 KB slot array, and + // their one reader was the `page_of` debug cross-check (which now + // skips Huge segments). Nothing else reads a Huge segment's slots past + // slot 1. With the strided stores gone, this is a contiguous 2 KB fill. let owner = page_off_for(1); let tab: *mut u32 = (&raw mut (*seg).page_off).cast(); - *tab.wrapping_add(1) = owner; - let mut j = 2; + let mut j = 1; while j < SLICES_PER_SEGMENT { - // Slices back to slot 1, which holds this huge block's page data. - (*seg).pages[j].slice_offset = (j - 1) as u16; - *tab.wrapping_add(j) = owner; + *tab.add(j) = owner; j += 1; } Ok((seg, block)) @@ -1090,6 +1146,23 @@ pub unsafe fn huge_free(seg: *mut Segment) -> Result<(), PrimError> { // Only chunk-multiple segments can reach an arena and need the // restore; a ragged one is released whole (or pooled, on wasm). let chunked = (*seg).total_size.is_multiple_of(SEGMENT_SIZE); + // With purging on, a freed huge block gives its pages back BEFORE its + // chunks rejoin the arena, exactly as `span_free` does for a span. + // It never did: arena memory is committed once and handed out as-is, + // so a dead 150 MB tensor stayed resident for the life of the process + // whatever `purge_delay` said — the unexplained ~150 MB of + // `docs/plans/huge-free-retention.md` (Whisper small's embedding is + // 152 MiB). Marked `purged_any`, so `restore_for_reuse` below + // re-commits it (checked since 2.2.3) and the next tenant gets + // demand-zero pages, not a decommitted range. A segment outside an + // arena is released whole below and needs none of this. + if chunked && crate::options::get(15) >= 0 && crate::arena::owns(seg.cast()) { + let base = seg.cast::().add(HEADER_SLICES * SEGMENT_SLICE_SIZE); + let bytes = (*seg).total_size - HEADER_SLICES * SEGMENT_SLICE_SIZE; + if os::purge(base, bytes, crate::options::is_enabled(5)).is_ok() { + (*seg).purged_any = true; + } + } let usable = !chunked || restore_for_reuse(seg); if chunked { if !usable { diff --git a/crates/rusty_alloc/src/segment_map.rs b/crates/rusty_alloc/src/segment_map.rs index f1f8f2f..2aa4009 100644 --- a/crates/rusty_alloc/src/segment_map.rs +++ b/crates/rusty_alloc/src/segment_map.rs @@ -273,6 +273,12 @@ pub fn register(seg: *mut Segment) { /// (OH-rusty_alloc-30). Reachable from a caller-supplied `manage_os_memory` /// range, so bounded here as well as there. #[cfg(all(not(ra_small_profile), not(all(target_arch = "wasm32", not(miri)))))] +/// +/// REFUTED (2026-10-07): coalescing the windows that share a bitmap word into +/// one `op` call (one locked RMW per word instead of per window) measured +/// +15 Ir on a 64 MiB malloc/free pair (824 -> 839): tracking the pending word +/// costs more instructions than the two RMWs it removes. It may still win on +/// cycles (a locked RMW is ~20), which callgrind cannot show; not kept. fn walk_windows(base: usize, size: usize, mut op: impl FnMut(usize, u32)) { let mut a = base; let end = base.saturating_add(size.max(1)); diff --git a/crates/rusty_alloc/tests/adopt_large_spans.rs b/crates/rusty_alloc/tests/adopt_large_spans.rs new file mode 100644 index 0000000..9b8c238 --- /dev/null +++ b/crates/rusty_alloc/tests/adopt_large_spans.rs @@ -0,0 +1,77 @@ +//! A segment abandoned with live LARGE blocks, whose blocks are then freed +//! from another thread, must be fully reusable once adopted. +//! +//! `docs/plans/huge-free-retention.md`, mechanism 1: a model loaded on one +//! thread, which then exited, and dropped on another kept ~0.6 of the model's +//! size resident and unreusable, whatever `purge_delay` said. Adoption retired +//! ONE dead large span per segment and left the rest to "later collects", but +//! a large span is never queued, so no collect ever reached it again. +//! +//! Own test binary: it reasons about which segments exist process-wide, and a +//! parallel test abandoning or adopting segments would muddy that. + +use rusty_alloc::alloc::{free, malloc, stats}; +use rusty_alloc::segment::USABLE_SLICES; +use rusty_alloc::types::SEGMENT_SLICE_SIZE; +use std::thread; + +/// A large (single-block) span at EVERY geometry: two slices is past +/// `MEDIUM_OBJ_SIZE_MAX` everywhere this crate builds (`tests/spans.rs`). A +/// literal 2.5 MB was a large span at the shipped 32 MiB segment and a HUGE +/// block, one segment each, under `ra_small_profile` and `ra_segment_size = +/// "256k"`, where this test then failed for a reason unrelated to its subject. +const SIZE: usize = 2 * SEGMENT_SLICE_SIZE; +/// Enough to FILL one segment (255 at the shipped geometry, 3 at the small +/// profile), like a model's weight matrices. Filling it is what makes the test +/// bite: with room left at the bump frontier, a reload fits even when the +/// adoption retired only one dead span, and the old behaviour would pass. +const N: usize = USABLE_SLICES / 2; + +fn alloc_all() -> Vec { + (0..N) + .map(|_| { + let p = malloc(SIZE); + assert!(!p.is_null()); + // SAFETY: a live block of SIZE bytes; touch it like loaded weights. + unsafe { core::ptr::write_bytes(p, 0x5A, SIZE) }; + p as usize + }) + .collect() +} + +// Not under Miri: the test needs a thread's EXIT to abandon its segments, and +// Miri runs against the mock prim, whose thread-exit (TLS destructor) hooks +// never fire (`prim/mock.rs`: "TLS destructors do not fire"). There the +// loader's segments are never abandoned, nothing adopts them, and the premise +// never occurs; Miri reported an assertion, not undefined behaviour. +#[cfg_attr(miri, ignore)] +#[test] +fn an_adopted_segment_gives_back_every_dead_large_span() { + // The loader allocates, then exits: its segment is abandoned with every + // block still live. + let blocks = thread::spawn(alloc_all).join().unwrap(); + // Another thread drops the "model": frees into the abandoned segment. + for p in blocks { + // SAFETY: live blocks from `alloc_all`, each freed once. + unsafe { free(p as *mut u8) }; + } + // A third thread loads the next one. It adopts the orphan, which now + // holds nothing live, so it needs no fresh segment at all. + let fresh = thread::spawn(|| { + let before = stats().segments; + let blocks = alloc_all(); + let fresh = stats().segments - before; + for p in blocks { + // SAFETY: as above. + unsafe { free(p as *mut u8) }; + } + fresh + }) + .join() + .unwrap(); + assert_eq!( + fresh, 0, + "the adopted segment's dead large spans were not all retired, so the \ + next load took {fresh} fresh segment(s) while their memory sat unused" + ); +} diff --git a/crates/rusty_alloc/tests/double_free.rs b/crates/rusty_alloc/tests/double_free.rs index a64591b..245e9ec 100644 --- a/crates/rusty_alloc/tests/double_free.rs +++ b/crates/rusty_alloc/tests/double_free.rs @@ -196,3 +196,51 @@ fn first_class_heap_abandoned_double_free_aborts() { "the second free of a first-class abandoned block RETURNED" ); } + +const LARGE_MARKER: &str = "RUSTY_ALLOC_LARGE_DOUBLE_FREE_CHILD"; + +/// A LARGE block (a single-block span) freed twice. Its span is now kept +/// carved for same-size reuse (`Heap::large_cache`), with `used == 0`; the +/// second free must find that and abort, where it used to reach a scrubbed +/// slot and abort by a different route. +#[cfg_attr(miri, ignore)] +#[test] +fn large_double_free_aborts() { + if std::env::var(LARGE_MARKER).is_ok() { + let n = 2 << 20; + let p = rusty_alloc::alloc::malloc(n); + assert!(!p.is_null(), "child: malloc failed"); + // SAFETY: p is live and ours. + unsafe { rusty_alloc::alloc::free(p) }; + // SAFETY: DELIBERATELY WRONG — the bug under test. + unsafe { rusty_alloc::alloc::free(p) }; + eprintln!("child: second free of a large block RETURNED"); + std::process::exit(97); + } + let exe = std::env::current_exe().expect("current_exe"); + let out = Command::new(exe) + .env(LARGE_MARKER, "1") + // `--nocapture`: the child's own line must reach `out.stderr`, which + // libtest's capture would otherwise swallow when the child aborts. + .args(["--exact", "large_double_free_aborts", "--nocapture"]) + .output() + .expect("spawn child"); + assert!( + !out.status.success(), + "a large double free was accepted silently" + ); + assert_ne!( + out.status.code(), + Some(97), + "the second free RETURNED: detection did not fire" + ); + // The exit code alone is not enough: when the second free returns, a debug + // build still dies later (the teardown flush retires the span twice and + // trips `debug_validate_segment`), so the abort must be the second free's. + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + !stderr.contains("second free of a large block RETURNED"), + "the second free RETURNED and something later aborted instead: +{stderr}" + ); +} diff --git a/crates/rusty_alloc/tests/huge_purge.rs b/crates/rusty_alloc/tests/huge_purge.rs new file mode 100644 index 0000000..d1266c0 --- /dev/null +++ b/crates/rusty_alloc/tests/huge_purge.rs @@ -0,0 +1,43 @@ +//! With purging on, a freed HUGE block must give its pages back to the OS. +//! +//! `docs/plans/huge-free-retention.md`, mechanism 3: `span_free` purges a +//! freed span, but `huge_free` returned a dead block's chunks to the arena +//! resident, whatever `purge_delay` said. A consumer that loaded Whisper small +//! (a 152 MiB embedding) kept ~150 MB over the system allocator with purging +//! on, for the life of the process. +//! +//! Own test binary: it reads the process's resident memory, which any test +//! running alongside would move. + +use rusty_alloc::alloc::{free, malloc}; +use rusty_alloc::stats::process_info; +use rusty_alloc::types::LARGE_OBJ_SIZE_MAX; + +fn rss() -> usize { + process_info().3 +} + +#[test] +fn a_freed_huge_block_is_purged_when_purging_is_on() { + rusty_alloc::options::set(15, 0); // purge_delay: purge at once + // Past LARGE_OBJ_SIZE_MAX, so a dedicated huge segment from the arena. + let size = 3 * LARGE_OBJ_SIZE_MAX; + let p = malloc(size); + assert!(!p.is_null()); + // SAFETY: a live block of `size` bytes; every page made resident. + unsafe { core::ptr::write_bytes(p, 0x5A, size) }; + let live = rss(); + // SAFETY: freed once. + unsafe { free(p) }; + let after = rss(); + rusty_alloc::options::set(15, -1); // restore the shipped default + if live == 0 { + return; // no RSS reporting on this platform (miri, wasm, bare metal) + } + let returned = live.saturating_sub(after); + assert!( + returned >= size / 2, + "freeing a {size}-byte huge block returned only {returned} bytes \ + (resident {live} -> {after}); its pages stayed in the arena" + ); +} diff --git a/crates/rusty_alloc/tests/large_cache.rs b/crates/rusty_alloc/tests/large_cache.rs new file mode 100644 index 0000000..bdf114f --- /dev/null +++ b/crates/rusty_alloc/tests/large_cache.rs @@ -0,0 +1,81 @@ +//! The per-heap large-span cache (`Heap::large_cache`): a freed large block's +//! span is kept for the next allocation of the same slice count, never for +//! another size, and handed back by a collect. +//! +//! Own test binary: the cache is per thread, and the assertions read this +//! thread's heap counters. + +use rusty_alloc::alloc::{collect, free, malloc, stats}; +use rusty_alloc::types::SEGMENT_SLICE_SIZE; + +/// A large span at every geometry (two slices; `tests/spans.rs`). A literal +/// 2 MiB is a HUGE block under `ra_small_profile`, which the cache never sees, +/// so the test passed there without testing anything. +const TWO_MIB: usize = 2 * SEGMENT_SLICE_SIZE; + +#[test] +fn same_size_reuses_the_span_other_sizes_do_not_and_collect_flushes() { + // Same size: the very same block comes back, no span carved for it. + let a = malloc(TWO_MIB); + assert!(!a.is_null()); + // SAFETY: live block; then freed once. + unsafe { core::ptr::write_bytes(a, 0x5A, TWO_MIB) }; + // The address alone cannot tell the cache from first-fit (a retired span + // is re-carved in the same place), but retirement can: without the cache + // the free retires the span at once. + let retired = stats().pages_retired; + // SAFETY: freed once. + unsafe { free(a) }; + assert_eq!( + stats().pages_retired, + retired, + "the freed large span was retired, not cached" + ); + let b = malloc(TWO_MIB); + assert_eq!( + b, a, + "a same-size large allocation did not reuse the cached span" + ); + // SAFETY: b is a live TWO_MIB block (the cached span, re-handed). + unsafe { core::ptr::write_bytes(b, 0x11, TWO_MIB) }; + + // A different size never takes the cached span: it must still be cached + // (and so not coalesced away) when `b` is freed and re-requested. + // SAFETY: freed once. + unsafe { free(b) }; + let c = malloc(TWO_MIB + SEGMENT_SLICE_SIZE); + assert!(!c.is_null()); + assert_ne!(c, a, "a larger allocation was handed the cached 2 MiB span"); + let d = malloc(TWO_MIB); + assert_eq!( + d, a, + "the cached span did not survive an allocation of another size" + ); + + // A RUN of large frees is a teardown: the second free in a row retires + // the cached span and its own, so nothing is left cached to be refilled + // out of address order by the next load (a same-thread model swap peaked + // at 1.04 models instead of 1.01 before this rule). + // SAFETY: c is live, freed once. + unsafe { free(c) }; // lone free after an allocation: cached + let retired = stats().pages_retired; + // SAFETY: d is live, freed once. + unsafe { free(d) }; // second in a row: flush both + assert_eq!( + stats().pages_retired - retired, + 2, + "a run of large frees left a span cached" + ); + + // A collect hands a cached span back to its segment. + let e = malloc(TWO_MIB); + assert!(!e.is_null()); + // SAFETY: e is live, freed once. + unsafe { free(e) }; // lone free: cached + let retired = stats().pages_retired; + collect(false); + assert!( + stats().pages_retired > retired, + "collect did not retire the cached large span" + ); +} diff --git a/crates/rusty_alloc/tests/spans.rs b/crates/rusty_alloc/tests/spans.rs index 95d5a19..f8c0695 100644 --- a/crates/rusty_alloc/tests/spans.rs +++ b/crates/rusty_alloc/tests/spans.rs @@ -42,10 +42,14 @@ fn span_lifecycle_and_realloc() { // SAFETY: live block freed once. unsafe { free(p) }; let s2 = stats(); + // Not retired: with purging off (the default) the heap keeps one freed + // large span carved for the next allocation of the same slice count + // (`Heap::large_cache`); a collect retires it. The reuse below is what + // this section is about either way. assert_eq!( s2.pages_retired - s1.pages_retired, - 1, - "large span not retired" + 0, + "freed large span was retired instead of cached" ); // "Similar size": slightly smaller than `l`, so it must reuse the span // just retired rather than take a fresh segment. diff --git a/docs/opps.md b/docs/opps.md index 54e1045..da6411c 100644 --- a/docs/opps.md +++ b/docs/opps.md @@ -352,6 +352,172 @@ arena could hand a 64 MiB pair's touched half out first. Backlog: needs the `retain` probe from the consumer's harness as its instrument, and a working set number, not an instruction count, as its verdict. +## Vein census, 2026-10-07 (where the next ten deterministic wins are) + +**Why a new census.** `bench/opscan.sh` (16 ops, two-point callgrind) now has +rusty_alloc AHEAD of mimalloc on every op it covers: small 52.25, med 56.50, +big 92.00, opscan "huge" (2 MiB) 293.00, calloc 93.19, batch 53.20, realloc +231.20, aligned 83.69, xthread 77.08 Ir/op. Four curiosity rounds mined those +paths; the oracle is no longer a target source there. The veins are on paths +opscan does not drive. A side probe (`F:/ra-corpus-tmp/veins/veins.c` + +`scan.sh`, same two-point method, pointers escaped through a `volatile` sink +because GCC elides an unused malloc/free pair, which first read a flat 30 Ir +for every allocator) measured them: + +| op | rusty_alloc | mimalloc 2.4.5 | glibc | +|---|---:|---:|---:| +| 64 MiB malloc + free (true huge) | **53,538** | 53,608 | 384 | +| thread life (spawn, 32 small allocs, exit) | **63,977** | 101,460 | 15,920 | +| exited thread's 8 large blocks freed elsewhere, then reloaded | **76,050** | 172,867 | 11,773 | +| calloc 1 MiB + free | 57,812 | 110,746 | 57,805 | + +Note: callgrind counts `rep stosb` once per BYTE, so memset-heavy rows overstate +cycles; the BYTES are still real (and first-touch faults with them). Rank by +bytes and calls, then confirm on whole-program Ir. + +**Ranked veins, biggest first:** + +1. **The 45 KB segment scrub.** `segment_alloc` and `huge_alloc` zero the whole + `Segment` (512 `Page` slots x 88 B) whenever the chunk is recycled + (`!mem_zero`), which after warm-up is always. It is 47 k of the 53.5 k Ir of a + huge pair (a huge segment uses ONE page slot) and 47 k of the ~48 k Ir a thread + life costs over glibc (each new thread's first segment is a recycled chunk). + The comment beside it already says carved slots are re-initialised by + `span_mark`. Lever: scrub the header fields and only the slots something reads + before a carve (huge: slots 0-1). Gate: `debug_validate_segment`, the spans and + adopt suites, miri. Ceiling: ~47 k Ir and 11 first-touched pages per segment. +2. **Large spans are carved and retired on every alloc/free.** The 2 MiB pair is + 293 Ir (big 64 KiB: 92), spread over `span_alloc` 54, `span_free` 52, + `malloc_generic_once` 49, `span_from_segments` 33, `free_local_at` 25; the + slice-marking loops scale with the span. Upstream keeps a retired page for + reuse; here "span reclamation IS the reuse mechanism". Lever: a one-entry + per-heap cache of the last retired large span, reused when the size class + matches. Ceiling: most of 293 -> ~100 on alloc/free-same-size loops (Vec + growth, tensor scratch, request buffers). +3. **Huge path, excluding the scrub:** ~4.7 k Ir per pair in `huge_alloc` + (2.1 k), `huge_free` (1.8 k) and a `range.rs` iterator inside `huge_free` + (0.8 k): per-slice `segment_map` register/unregister loops that scale with the + block (64 MiB = 1,024 slices). Lever: register per chunk, not per slice. +4. **Thread start/exit beyond the scrub:** `malloc_generic_walk` ~4.2 k Ir per + thread (first allocations walk empty bins), `collect_inner` ~2 k at exit, + `span_free` ~1.5 k, `__nptl_deallocate_tsd` 0.7 k. Thread pools (tokio's + blocking pool, rayon) pay this per spawn. +5. **`realloc_live` self cost: 65 Ir/op** before the copy, more than a whole + small alloc/free pair; plus 17 from inlined page.rs and 10 from segment.rs. + Per-line census first (opps #2 shaped this path once already). +6. **calloc wrapper:** 93.19 vs malloc's 52.25 per pair at 256 B; the memset is + 26, leaving ~15 Ir of wrapper and `free_is_zero` bookkeeping. Opps #5 + (`zero_block` re-resolving `usable_size`) is still OPEN and lives here. +7. **Adoption rescan (guard for the 2026-10-07 fix).** `adopt_segment` now + restarts its slice walk after each dead large span it retires: O(spans^2) + per adoption. Cheap at 8 spans; measure a segment of 64+ dead spans before + shipping, and resume from the merged span instead of slice 0 if it shows. +8. **Duplicate arena scans.** `huge_free` with purging on now calls + `arena::owns` and then `chunk_free_n`, two linear scans of the arena table; + `segment_free` likewise on its failure arm. Fold into one lookup returning the + arena id. Small, but it is per huge/segment free. +9. **Real programs, re-profiled.** perl/sqlite/python/lua/jq whole-program Ir + (baselines in `memory/wsl-icount-rig.md` were set before 2.2.x); re-run the + exact by-object census (`objir.awk`) and rank functions, not ops. This is + where a vein that no synthetic op shapes shows up first. +10. **Bytes, not instructions.** `tools/wasm-size.sh` and the firmware rig + measure code size deterministically; every fix above adds code, and the + small profile's flash budget is a currency too. Re-baseline, and look for + monomorphised duplicates in the per-arm `GlobalAlloc` paths. + +### Vein census results, 2026-10-07 (working tree, uncommitted) + +**Read this first: a memset-heavy row overstates cycles under callgrind.** +glibc's memset uses `rep stosb` above 2 KB and callgrind counts it once per +BYTE. Every number below marked *honest* was taken with +`GLIBC_TUNABLES=glibc.cpu.x86_rep_stosb_threshold=0x7fffffff:glibc.cpu.x86_rep_movsb_threshold=0x7fffffff`, +which makes memset/memcpy vector loops that callgrind counts like any other +code. Quote the honest column. Also: GCC deletes an unused malloc/free pair at +`-O2`; a probe must escape the pointer (a `volatile` sink) or every allocator +reads the same flat count. + +| change | probe | before | after | delta | +|---|---|---:|---:|---:| +| **vein 1** scrub only the owner table + slots read before a carve; zero a slot at its first carve | 64 MiB malloc+free (honest) | 8,959 | 884 | **-90.1 %** (with vein 3) | +| | thread life (honest) | 19,387 | 17,409 | **-10.2 %** | +| | Rust `threads` workload, 20,000 steps (honest, checksum equal) | 3,191,866 | 2,713,575 | **-15.0 %** | +| **vein 3** huge: no strided `slice_offset` writes to slots 2..511, owner table scrubbed once, `wait_no_remote_in_flight` scans slot 1 only | (inside the 64 MiB row) | 2,931 | 884 | | +| **vein 2** per-heap one-entry large-span cache (`Heap::large_cache`), newest-first, <= 4 MiB, purging off only, flushed by every collect | 2 MiB malloc+free | 293.05 | 126.07 | **-57.0 %** | +| | exited-thread reload, 8 x 2.5 MB (honest) | 30,738 | 30,946 | +0.7 % (cache bookkeeping) | +| **retention fix** adoption retires every dead large span | 64 dead spans adopted (honest) | 136,899 | 139,729 | +2.1 % (the 63 spans 2.2.3 stranded) | + +Unchanged to the instruction on every opscan op (small 52.25, med, big 92.00, +calloc 93.19, realloc 231.20, aligned 83.69, mixed 92.70, xthread 77.08). +Real programs, allocator-only Ir: perl +203 of 24.6 M, sqlite +90 of 4.6 M +(mimalloc's own perl count moved +172 between the same two runs: session +pedestal, not the change). Rust `maps`/`trees`/`overaligned` flat. Code size: ++1,936 bytes `.text` on the Rust workloads (+0.6 %). + +**Refuted, recorded:** keeping the OLDEST freed span in the large cache cost ++882 Ir per op on the reload probe (later frees coalesced to its right and +retiring it last re-marked the whole run); newest-first fixed it. Resuming the +adoption walk at the merged span instead of slice 0 measured flat at 64 spans +(kept to bound the worst case, not claimed). A first cache-test design proved +nothing: first-fit returns the same address without the cache, so the test now +reads `pages_retired`; the large double-free test needed `--nocapture` in its +child, or libtest swallowed the line that proves which free aborted. + +**Still open from the census:** `realloc_live` frame (13 Ir/call of entry/exit), +calloc wrapper (opps #5), the multi-chunk arena scan (bit-at-a-time, ~166 Ir +per huge allocation), `segment_map` per-window RMWs, re-profiling real programs +by function, code size. + +### Vein census round 2 + validation sweep, 2026-10-07 + +| change | probe | before | after | +|---|---|---:|---:| +| multi-chunk arena claim: bitmap word loaded once per word, a one-word run claimed with one masked `fetch_or` (and its dirty bits with another) | 64 MiB malloc+free | 884 | **824 (-6.8 %)**, 6 -> 2 locked RMWs per 3-chunk claim | +| `owns`/`chunk_free`/`chunk_free_n` share one scan (`find_arena`) | same | 824 | 824 (neutral; -3 B wasm gz) | +| large cache: cache only a LONE large free (a bool `large_freed`), retire the cached span first on every large free | same-thread model swap, purge off | 1.04 models (the first cache) | **1.01** (= no cache) | +| | 2 MiB pair | 293.05 (2.2.3) | 138.06 (-52.9 %) | +| | exited-thread reload, honest | 31,472 (2.2.3) | 30,205 (-4.0 %) | +| | thread life, honest | 19,387 (2.2.3) | 16,988 (-12.4 %) | + +Final build vs 2.2.3 on opscan: small/aligned/calloc/realloc/xthread +identical, big 92 -> 91, mixed 92.70 -> 92.04. Real programs, allocator-only +Ir: perl 24,633,532 -> 24,630,214, sqlite 4,582,668 -> 4,582,578. Endless Sky +start-up: output identical, allocator Ir 57,121,237 -> 57,119,496. + +**Refuted, recorded (do not retry the wrong way):** +- `segment_map::walk_windows` coalescing windows per bitmap word: +15 Ir on + the 64 MiB pair (tracking the pending word costs more than the two RMWs). +- `chunk_free_n` clearing a one-word run with one masked `fetch_and`: +9 Ir + for a 3-chunk free. +- Flushing the large cache on a size miss in `large_alloc`: did NOT fix the + model-swap spill, and inlined into `malloc_generic_once` it cost every + generic op +3 Ir (big 92 -> 95, mixed +2.2). Bisected with a per-function + diff; two earlier guesses (the shared `free_local_at` path, a new `u32` + field changing `Heap`'s layout) were wrong. +- calloc (opps #5) and `realloc_live`: at their floor; #5's re-resolution is + already gone, and realloc's frame is the price of inlining malloc + memcpy + + free (its outlining was refuted +12 Ir in 2026-08). + +**Validation sweep (final code):** workspace `--all-features` 177/177 (Windows); +`corpus/linux-gates.sh` 42 suites / 0 failures; embedded matrix 15/15 +(small profile, 256k segments, single-threaded, aligned region, both RISC-V +bare-metal targets x both geometries, the no_std refusal); clippy per feature; +wasm self-test + size (20,873 B gz, +361 B vs 2.2.3, inside the +3 % ratchet); +gate self-test 11/11; semgrep clean; miri gate clean (the thread-exit test is +`ignore`d under miri: the mock prim fires no TLS destructors); mimalloc-bench +sweep 19/19; churn 5/5; `corpus/realworld.sh`: jq, sqlite, python, git, xz, +zstd, lua, perl byte-identical output across ra/mi/sys (imagemagick is +nondeterministic on every arm; redis is jemalloc-linked and fails under ANY +preloaded allocator, 2.2.3 included); alloc-eval `check` on all 8 traces +(~37.5 M ops) and the 28-thread battle replay with purging on and off. + +**Edge cases the sweep caught and fixed:** the large cache's same-thread +model-swap spill (above); `adopt_large_spans`/`large_cache` hard-coded sizes +that are HUGE blocks under `ra_small_profile`, so they failed (or passed +vacuously) there, now geometry-derived and poison-checked at both geometries; +a `let _ = retire_span(..)` missing its same-line terminal comment (semgrep +`discarded-lifecycle-result`, CI-blocking); `corpus/linux-gates.sh` +hard-coded `/mnt/c/...` and exited 1 before running anything. + ## Banked ### 8. Collect-loop double `block_next` — LANDED 2026-08-20 diff --git a/docs/plans/huge-free-retention.md b/docs/plans/huge-free-retention.md new file mode 100644 index 0000000..05d984e --- /dev/null +++ b/docs/plans/huge-free-retention.md @@ -0,0 +1,209 @@ +# Freed huge allocations stay resident in a long-lived service + +**Status:** measured in a shipping consumer; **three mechanisms found, two fixed in the working tree 2026-10-07 (uncommitted, see §7)**; asks 1-3 and 5 open · +**Date:** 2026-10-07 · **Seen on:** 2.2.3, default options (`purge_delay = -1`) · +**Platform:** measured on Windows 11 x86-64; the consumer runs on Linux (Fly.io, 4 GB VM) · +**From:** the ragconverter session (`rag-converter`, `crates/ragconverter-server`) + +--- + +## 1. What happened + +ragconverter's API was OOM-killed twice in five minutes on 2026-10-07. Two requests were in +flight on a 4 GB machine. Every request in flight at each kill failed with a 502. + +The server loads two models of about 1 GB each, built from safetensors through candle: +- Whisper small.en: 922 MB of f32; +- SmolVLM-256M: 978 MB once its bf16 weights are widened to f32. + +The ragconverter side of the fix is to keep only one model resident at a time and drop the other +before loading it. That fix did not bring the peak down by the size of a model: freed model +memory stayed resident under rusty_alloc. With the system allocator it went back to the OS. + +## 2. Measurement + +One process converts a video (all engines), a FLAC and a PNG, one job at a time. Each job loads +the model it needs and drops the other. The measure is the process's peak working set, sampled +every 200 ms. It is a release build with `opt-level=3` and the same inputs every time. + +| Allocator / option | Peak working set | Runs | +|---|---:|---| +| system allocator (no `#[global_allocator]`) | **2,722 MB** | 1 | +| rusty_alloc 2.2.3, default (`purge_delay = -1`) | **3,802 / 3,832 / 3,881 MB** | 3, interleaved with the next row | +| rusty_alloc 2.2.3, `RUSTY_ALLOC_PURGE_DELAY=0` | **2,868 / 2,885 / 2,887 MB** | 3, interleaved | +| `RUSTY_ALLOC_PURGE_DECOMMITS=1` (purge still off) | 3,472 MB | 1 | +| `PURGE_DELAY` = 10 / 250 / 1000 | 2,797 – 2,900 MB | 2 each | + +- **Retention:** about **0.95 GB**, roughly one dropped model, against the same allocator with + purging on. That is about 1.1 GB against the system allocator. +- **Consistency:** the gap is the same in all three interleaved pairs. +- **Cause:** it is the purge setting and not the workload. Any `purge_delay >= 0` closes most + of it. The remaining ~150 MB over the system allocator has not been explained (see §4, + ask 4). + +**Time is not settled.** In the three interleaved pairs, `purge_delay = 0` took 48–50 s against +38–40 s for the default, about 25% slower every time. But a headless browser test was running +on the same machine for part of those runs, and the delay sweep's timings range from 49 s to +183 s. Read the 25% as a lead to re-measure on a quiet machine, not as a result. + +## 3. Why it matters beyond this consumer + +This matches what the LEDGER already says from the soak (§ "Actionable consequence"): +*long-lived services should set `purge_delay >= 0`*. The difference is the shape: +- the soak saw ~650 MiB against a ~175 MiB live set, drifting slowly; +- here **one free of a gigabyte-scale buffer** decides whether the next allocation fits in the + machine. + +Any service that loads and swaps models, or holds one large request buffer at a time, hits this +on its first swap rather than after hours. The failure is an OOM kill, not a slow climb. + +The consumer cannot easily opt in either: +- `ragconverter-alloc::configure(Profile)` is a no-op. The crate exposes no global purge or + configure, and `Heap::collect` is per-heap, which a `#[global_allocator]` cannot reach. +- So the only lever is the environment variable, set before the first allocation, which a + deployment has to know to set. + +## 4. Asks, most useful first + +1. **Release huge-segment frees to the OS whatever `purge_delay` is.** + - A single allocation of tens of megabytes or more is the case where retention costs most + and reuse is least likely. + - It is also the case `purge_delay = -1` was not chosen for: the M8 defect is about span + purge and recommit, and huge segments are freed whole. + - Worth checking against `oracle/mimalloc`, which this was not compared with, to see what + it does with a freed huge segment. +2. **A global purge-now call a `#[global_allocator]` consumer can make**, e.g. + `rusty_alloc_api::purge()`: collect every heap, including abandoned ones, and purge free + spans now. A consumer knows exactly when it has dropped a gigabyte. Calling it then costs + nothing on the hot path, unlike `purge_delay = 0` (see the timing note in §2). The + ragconverter-alloc seam is already waiting for this. +3. **Options settable from code, not only the environment**, so a deliverable can say + "long-lived service, purge on" in one place: `configure(Profile::LongLived)`. +4. **Explain the remaining ~150 MB** over the system allocator with purging on. A candidate: + huge pages freed from a thread other than the one that allocated them sit DELAYED + (`alloc.rs`, "huge pages sit DELAYED"). Candle builds tensors on one tokio blocking-pool + thread and the model can be dropped on another. That thread may never allocate again, and + may exit holding the delayed free. This is not verified: no counter was read. +5. **Re-measure the cost of `purge_delay = 0` on a quiet machine** with an inference-shaped + workload (many medium alloc/free cycles around a large resident set). If the 25% is real, + it is probably decommit/recommit churn on medium spans. That would make ask 1 or ask 2 + the right fix rather than turning eager purging on globally. + +## 5. Reproduce + +From the rag-converter repo, with the weights staged in the `/opt/models` layout: + +```sh +cargo test -p ragconverter-server --release --no-run +# then run the ignored rig and sample the process's peak working set / VmHWM: +RAGCONVERTER_AUDIT_DIR= RIG_FILES=video-h264.mp4,audio.flac,card.png \ +CARMENTA_WEIGHTS= FFAI_MODELS= HEAVY_JOBS=1 \ + target/release/deps/ragconverter_server-*.exe --ignored --exact routes::tests::concurrent_heavy_pair +``` + +- **Allocator arms:** set `RUSTY_ALLOC_PURGE_DELAY` for the option arms. Comment out + `#[global_allocator]` in `crates/ragconverter-server/src/main.rs` for the system-allocator arm. +- **Measuring on Linux:** read `VmHWM` from `/proc//status` at exit, instead of sampling. + +## 6. What ragconverter does meanwhile + +- **Purge on in production:** sets `RUSTY_ALLOC_PURGE_DELAY=0` on the Fly machine, following + the LEDGER's own recommendation for long-lived services. The M8 crash is Windows-first (Linux + `MADV_DONTNEED` does not fault on touch), and Fly is Linux. +- **One model at a time:** keeps one big model resident and runs heavy jobs one at a time. +- **Idle unload:** unloads idle models. +- **Alerts:** alerts on high memory and on an unclean restart. + +If ask 1 or ask 2 lands, ragconverter can drop the environment variable and call purge exactly +when it drops a model. + +## 7. Findings (2026-10-07) + +A model-swap probe reproduced it without candle: a "model" is Whisper small's +tensor-size mix (one 152 MiB embedding, 2.4-9.4 MB matrices, 3-12 KB vectors), +909 MB in all, touched page by page. Load model A, drop it, load model B, drop +it; read the process's peak working set. Windows 11, release build. Four +shapes of who loads and who drops: + +| shape | system | mimalloc 2.3 | rusty_alloc 2.2.3 default | 2.2.3, `PURGE_DELAY=0` | fixed tree, default | fixed tree, `PURGE_DELAY=0` | +|---|---:|---:|---:|---:|---:|---:| +| one thread loads, drops, reloads | 1.00 | 1.01 | 1.01 | 1.01 | 1.01 | 1.01 | +| T1 loads and exits, T2 drops, T3 reloads | 1.00 | 1.04 | **1.62** | **1.62** | 1.01 | 1.01 | +| T1 loads and drops, exits; T2 reloads | 1.00 | 1.04 | 1.01 | 1.01 | 1.01 | 1.01 | +| T1 loads, drops, stays alive with small leftovers; T2 reloads | 1.00 | 1.04 | **1.23** | 1.01 | **1.23** | 1.01 | + +(peak working set in models; the probe is `F:/ra-corpus-tmp/huge-ret`.) +Memory still resident at exit with purging on: 2.2.3 **159 MB**, fixed tree +**7 MB**, system 4-5 MB. + +**Three mechanisms, not one.** + +1. **Dead large blocks in an abandoned segment were never retired** (fixed). + A thread that exits abandons its segments with their blocks live. Blocks + freed into them later are collected when another thread adopts the + segment, but `adopt_segment` retired only ONE dead large span per segment + and `break`, "leaving the rest to later collects"; a large span is never + queued, so no collect reaches it again. The rest stayed carved and + resident for the life of the process: +0.62 of a model, and **purging did + not help** (nothing reached `span_free`). Upstream collects every + concurrent free of an abandoned segment (`mi_segment_check_free` in + `mi_segment_try_reclaim`) and scored 1.04. Now every dead large span is + retired, rescanning after each because retiring coalesces. Fixing it + exposed a second slip, also fixed: when those retires emptied the + segment, `retire_span` parked it as the heap's one empty segment and the + tail of `adopt_segment` then counted it again and released it, leaving + `empty_segments == 1` with no empty segment behind it. + Test: `tests/adopt_large_spans.rs`; with the old `break` it fails (the + next load takes a fresh segment). + +2. **A live, idle thread's segments strand free space** (by design; purging + is the remedy). Small pages are carved from the newest segment, so a + model's small tensors spread across nearly every segment the load used, + and each emptied small page stays as its bin's reuse cache, pinning a 32 + MiB segment to a thread that may sit idle (a tokio blocking-pool thread + does, for 10 s by default, and longer under load). Measured: 26 segments + allocated, 19 released, 7 (224 MiB) kept; +0.23 of a model. Free spans are + per-thread by design; with purging on they are not resident, which is why + `PURGE_DELAY=0` closed most of the consumer's gap. mimalloc packs small + pages more tightly and scored 1.04 with its purging on or off; changing + span placement is a separate, measured project. + +3. **A freed huge block (> 32 MiB) stayed resident whatever `purge_delay` + said** (fixed). `span_free` purges a freed span when purging is on; + `huge_free` returned a dead block's chunks to the arena, which holds + memory committed and resident. This is the consumer's unexplained ~150 MB + (ask 4): Whisper small's embedding is 51,865 x 768 x 4 = 152 MiB. With + purging on, `huge_free` now purges the block before its chunks rejoin the + arena, marks it `purged_any`, and `restore_for_reuse` re-commits it + (checked since 2.2.3), so the next tenant gets demand-zero pages. Test: + `tests/huge_purge.rs`; with the purge disabled it fails ("returned only 0 + bytes"). + +Verified on Windows: the whole `rusty_alloc` suite with and without `secure`, +clippy `-D warnings` (`secure`), fmt, `cargo check --target +wasm32-unknown-unknown`, unsafe census unchanged (973). + +**On the asks:** + +- **Ask 1 (release huge frees whatever `purge_delay` is):** done for + `purge_delay >= 0`. At `-1` it is a policy decision for the maintainer: + mimalloc purges huge segments by default because its purge is ON by default + (10 ms); ours is opt-in. Retaining a dead 150 MB block at `-1` saves a + re-fault only if the next huge allocation reuses it. +- **Ask 2 (a global purge-now call):** not done. It is what mechanisms 1 and + 2 still need at `-1`, and what memory freed into an exited thread's + segments needs until some thread allocates: the probe's "T1 loads and + exits, T2 drops" case ends with B's 915 MB resident until the next + allocation adopts it. +- **Ask 3 (options from code):** already possible, undiscoverably: + `rusty_alloc::options::set(15, 0)` sets `purge_delay` at run time, and + `span_free` reads it on every free, so it takes effect at once. Neither the + `rusty_alloc-api` seam nor a named constant exposes it. +- **Ask 4:** explained by mechanism 3 above; the DELAYED-huge-page candidate + was not needed. +- **Ask 5 (the 25 % cost of `purge_delay = 0`):** not re-measured. + +**For ragconverter, today:** keep `RUSTY_ALLOC_PURGE_DELAY=0` (or 10). On +2.2.3 it still loses ~150 MB per freed embedding-sized tensor and up to ~0.6 +of a model when a loader thread exits before the model is dropped; both are +gone in the fixed tree. diff --git a/tools/unsafe-baseline.txt b/tools/unsafe-baseline.txt index f8c4f0d..fc46944 100644 --- a/tools/unsafe-baseline.txt +++ b/tools/unsafe-baseline.txt @@ -1,6 +1,6 @@ 106 crates/rusty_alloc/src/alloc.rs - 16 crates/rusty_alloc/src/arena.rs - 77 crates/rusty_alloc/src/heap.rs + 15 crates/rusty_alloc/src/arena.rs + 80 crates/rusty_alloc/src/heap.rs 42 crates/rusty_alloc/src/init.rs 2 crates/rusty_alloc/src/lib.rs 17 crates/rusty_alloc/src/options.rs @@ -13,7 +13,7 @@ 6 crates/rusty_alloc/src/prim/wasm.rs 34 crates/rusty_alloc/src/prim/windows.rs 2 crates/rusty_alloc/src/random.rs - 43 crates/rusty_alloc/src/segment.rs + 45 crates/rusty_alloc/src/segment.rs 3 crates/rusty_alloc/src/stats.rs 16 crates/rusty_alloc_api/src/lib.rs 15 crates/rusty_alloc_bench/src/kernels.rs diff --git a/tools/wasm-size-baseline.txt b/tools/wasm-size-baseline.txt index 1d8af61..e45538e 100644 --- a/tools/wasm-size-baseline.txt +++ b/tools/wasm-size-baseline.txt @@ -1 +1 @@ -20321 +20873 From 47a7218a1814cd6b37cc3be4c91f7c75dfe1f320 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 7 Oct 2026 18:17:48 -0700 Subject: [PATCH 2/2] fix(alloc): fire the deferred-free hook with no &mut Heap alive The heartbeat in Heap::malloc_generic_body called options::deferred_free while malloc_generic's `&mut self` was live and protected. A hook that allocates (the openheimer test's) or frees re-entered the same heap through a raw pointer under that borrow: undefined behaviour under Stacked Borrows. Miri has reported it on every main run since 2.2.3, which kept the CI badge red. The heartbeat now only marks the hook DUE (Heap::deferred_due, set when options::deferred_registered() peeks a hook). Every allocation entry in alloc.rs fires a due hook BEFORE forming &mut Heap (fire_deferred_if_due); malloc_slow keeps both arms in tail position, jumping to the cold malloc_slow_deferred when due. Upstream fires the hook at the start of the generic path; here it runs at the start of the next entry, within the hook's "called occasionally, with a heartbeat" contract. Cost, measured: big 91 -> 94, mixed 92.04 -> 94.26 Ir/op; perl/sqlite allocator Ir +0.16 % / +0.22 %. The first shape (a call inside malloc_slow) gave it a frame and cost big +9; refuted, recorded. Miri: tests/openheimer.rs now passes, 37/37 (it aborted at the first UB before, so the suites after it had not run under Miri in CI since 2.2.3). Unsafe census 977 -> 981 (UNSAFE.md). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/rusty_alloc/CHANGELOG.md | 9 +++ crates/rusty_alloc/UNSAFE.md | 2 +- crates/rusty_alloc/src/alloc.rs | 101 +++++++++++++++++++++++++++--- crates/rusty_alloc/src/heap.rs | 17 ++++- crates/rusty_alloc/src/options.rs | 12 +++- tools/unsafe-baseline.txt | 2 +- 6 files changed, 130 insertions(+), 13 deletions(-) diff --git a/crates/rusty_alloc/CHANGELOG.md b/crates/rusty_alloc/CHANGELOG.md index ac365a5..e902283 100644 --- a/crates/rusty_alloc/CHANGELOG.md +++ b/crates/rusty_alloc/CHANGELOG.md @@ -23,6 +23,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 embedding (152 MiB) stayed resident for the life of the process. Details, measurements and the consumer report: `docs/plans/huge-free-retention.md`. +- **A deferred-free hook that allocates is no longer undefined behaviour.** + `mi_register_deferred_free`'s hook fired inside the generic allocation + path while that path held `&mut Heap`, so a hook that allocated (or freed) + re-entered the heap under a protected exclusive borrow. Miri reported it + (Stacked Borrows, `tests/openheimer.rs`), and it had kept the `miri` CI job + red since 2.2.3. The hook now fires at the next allocation entry, before + any `&mut Heap` exists. Cost: one flag test per slow-path entry (perl and + sqlite allocator instructions +0.16 % / +0.22 %). + ### Performance - **Recycled segments are no longer scrubbed whole.** Every thread's first diff --git a/crates/rusty_alloc/UNSAFE.md b/crates/rusty_alloc/UNSAFE.md index b8d8778..89c7134 100644 --- a/crates/rusty_alloc/UNSAFE.md +++ b/crates/rusty_alloc/UNSAFE.md @@ -20,7 +20,7 @@ unsafe in DEPENDENCIES, and ours are `libc` plus bindings-only `windows-sys`. | Module | Count | What the `unsafe` is for | Last audit | |---|---:|---|---| -| `alloc.rs` | 94 | The public entry points: raw-pointer reads on the malloc fast path (must never form `&mut` on the shared empty-heap sentinel), pointer-derived metadata on the free path (`segment_of`/`page_of`), block-content copies in the realloc family. **+15 on 2026-08-22**: the free path's two `asm!` sites — the memory-destination `used--` whose flags drive the retire branch (its `label` block is a separate item and carries its own `unsafe`), and the fused `cmp {tid}, fs:0` in both `free_inline` and `free_general` — plus `malloc_or`/`malloc_or_slow`, which give `operator new` a fast path whose miss is a tail call. Each asm reads or writes exactly one field it already had a valid pointer to; none widens what the surrounding code could already touch. **+2 on 2026-09-24 (instruction-count campaign, LEDGER):** `realloc` split into an inline null test and `unsafe fn realloc_live(p: NonNull, ..)` — one signature, and the `NonNull::new_unchecked` sits inside the block that already called it, on a pointer the line above proved non-null. The body is the old one unchanged; nothing new is dereferenced. **+10 on 2026-09-24 (round two, LEDGER):** (a) `realloc_move`, the moving arm shared by the growth and the shrink — one `unsafe fn`, its body block, and the two call-site blocks: the old inline move code, now reached with the copy length each arm already holds; (b) `zalloc` in `malloc`'s raw-read shape — its fast-path block reads through `hb`, which may be the sentinel, exactly as `malloc` does (no `&mut`, no write unless a block was popped, which proves the heap is real), and the cold `unsafe fn zalloc_slow` that performs the sentinel test the fast path dropped (signature + block); (c) `free_local_owned` / `free_local_no_xheap`, `owner_heap` split so its heap-creating fallback is a tail call — two signatures and their two blocks, the same loads and the same `free_local_at` as before; (d) `malloc_aligned_pow2`, an `unsafe fn` only because its caller must have proven the alignment a power of two (a violation is not memory-unsafe — the bound test still refuses a mask at or above half a segment). Nothing new is dereferenced anywhere; each carries its SAFETY line | 2026-09-24 | +| `alloc.rs` | 110 | The public entry points: raw-pointer reads on the malloc fast path (must never form `&mut` on the shared empty-heap sentinel), pointer-derived metadata on the free path (`segment_of`/`page_of`), block-content copies in the realloc family. **+15 on 2026-08-22**: the free path's two `asm!` sites — the memory-destination `used--` whose flags drive the retire branch (its `label` block is a separate item and carries its own `unsafe`), and the fused `cmp {tid}, fs:0` in both `free_inline` and `free_general` — plus `malloc_or`/`malloc_or_slow`, which give `operator new` a fast path whose miss is a tail call. Each asm reads or writes exactly one field it already had a valid pointer to; none widens what the surrounding code could already touch. **+2 on 2026-09-24 (instruction-count campaign, LEDGER):** `realloc` split into an inline null test and `unsafe fn realloc_live(p: NonNull, ..)` — one signature, and the `NonNull::new_unchecked` sits inside the block that already called it, on a pointer the line above proved non-null. The body is the old one unchanged; nothing new is dereferenced. **+10 on 2026-09-24 (round two, LEDGER):** (a) `realloc_move`, the moving arm shared by the growth and the shrink — one `unsafe fn`, its body block, and the two call-site blocks: the old inline move code, now reached with the copy length each arm already holds; (b) `zalloc` in `malloc`'s raw-read shape — its fast-path block reads through `hb`, which may be the sentinel, exactly as `malloc` does (no `&mut`, no write unless a block was popped, which proves the heap is real), and the cold `unsafe fn zalloc_slow` that performs the sentinel test the fast path dropped (signature + block); (c) `free_local_owned` / `free_local_no_xheap`, `owner_heap` split so its heap-creating fallback is a tail call — two signatures and their two blocks, the same loads and the same `free_local_at` as before; (d) `malloc_aligned_pow2`, an `unsafe fn` only because its caller must have proven the alignment a power of two (a violation is not memory-unsafe — the bound test still refuses a mask at or above half a segment). Nothing new is dereferenced anywhere; each carries its SAFETY line. The baseline had drifted to 106 without a row edit; recorded now. **+4 on 2026-10-07 (the Miri Stacked Borrows failure on `main`):** the deferred-free hook used to fire inside `Heap::malloc_generic`, where `&mut self` is protected, so a hook that allocates re-entered the heap through a raw pointer under that borrow. It now fires at the allocation entries before any `&mut Heap` exists: `fire_deferred_if_due` (an `unsafe fn` and its block: a raw read and write of the heap's own `deferred_due` flag, then the hook) and `malloc_slow_deferred` (an `unsafe fn` and its block: the same, then the `malloc_generic` call `malloc_slow` already made). The nine entry blocks that call it existed before; nothing new is dereferenced | 2026-10-07 | | `heap.rs` | 80 | Owner-thread page/queue manipulation under raw pointers (no two `&mut Page` may coexist), the aligned-allocation peek, span carving. **+3 on 2026-08-19**: `try_unlink_huge_segment` split out of `remove_huge_segment`. **+15 on 2026-08-22**: `malloc_generic` split into a small entry plus `malloc_generic_walk`, with `grow_front`, `try_guarded` and `drain_delayed` as cold arms — each split adds an `unsafe fn` signature and its block while dereferencing nothing the single function did not — and the immortal `EMPTY_DELAYED` sentinel that lets the heartbeat read its list without a null test. **+2 on 2026-09-07 (P4e, §2.15 of `docs/plans/small-metal.md`)**: the reclamation fixes — one `unsafe` for the periodic `generic_collect` sweep and one for the reclaim-and-retry that runs before `malloc_generic` returns null. Both call `collect_inner`, which allocates nothing and touches only this heap’s own pages on the owner thread; each carries its SAFETY line. `malloc_generic` itself became a safe wrapper over the renamed `malloc_generic_once`, so the split added no signature. **+2 on 2026-09-08:** the medium collect-and-retry ahead of the heartbeat -- one `unsafe` around `page_collect` + `page_pop` on the bin queue front, one reading `free_is_zero` off the page just popped. Both are the SAME operations `malloc_generic_walk` performs on the same page a few lines later, on the owner thread under the heap lock: the block moves earlier, nothing new is dereferenced. Each carries its SAFETY line. **+2 on 2026-09-16 (Openheimer):** the two refuse arms in `malloc_aligned_at_slow` when `bins::aligned_at_from` reports that `block + offset + align` would wrap for a caller-chosen offset — each is one `free_local` of the block just allocated on this thread, which has not escaped. **+1 on 2026-09-24 (instruction-count campaign, LEDGER):** the medium collect-and-retry's `page_collect` + `page_pop` block became two blocks so the `saw_remote_free` latch is stored between them — same two calls on the same page, owner thread, in the same order; nothing new is dereferenced. **+1 on 2026-09-24 (round two):** a medium miss grows the front page directly — one block around the `capacity < reserved` test and the `grow_front` call, on the same page the block above just collected, which is `grow_front`'s whole contract **+2 on 2026-09-25 (CURIOSITY ROUND FOUR):** the medium arm pops the front page BEFORE collecting it — one `page_pop` block and the `free_is_zero` read beside it, the same two operations the collect-then-pop below already performs on the same owner-thread page. **+1 on 2026-10-07 (`docs/opps.md` vein census, vein 2):** `collect_inner` hands the per-heap cached large span (`Heap::large_cache`) back with `retire_span` — one block around `segment_of` on the cached page's own address and the same `retire_span` a large free performed before the cache existed, on the owner thread. **+2 the same day:** `free_large_span` — an `unsafe fn` and its block — is the large-span arm of `free_local_at` moved into its own function (double-free check, cache, free-run flag); the same operations on the same page, owner thread | 2026-10-07 | | `init.rs` | 42 | Thread/heap lifecycle: the initial-exec TLS slot (`global_asm!` + fs-relative asm reads), thread-pointer register reads (`fs:0`/`gs:0x30`/`tpidrro_el0`), heap-box creation/teardown, the abandonment path run inside platform TLS destructors. **+1 on 2026-09-09 (`firmware-what-is-left.md` §7.3): an ATTRIBUTE, not an operation** — `unsafe(link_section = ".rodata.…")` on `EMPTY_HEAP_BOX`, bare metal only, so the never-written sentinel lives in flash instead of costing 1,752 bytes of RAM. The token is counted because the census is a text search; the contract it rests on (raw reads only, no page ever stores its address) is the one the `Sync` impl below already carries, and a write would now fault against the flash cache instead of silently landing. `create_heap`'s template copy from it is a `ptr::read` inside the block that already existed. **+5 on 2026-09-16 (Openheimer, OH-rusty_alloc-13):** thread exit now abandons EVERY heap the dying thread still owns, not only the one in `done_slot` — a first-class heap that was never `heap_delete`d, or one installed with `set_default_heap`, used to stay DELAYED under a dead owner forever. `take_heap_owned_by` unlinks one such box under `HEAPS_LOCK` (one block: the registry walk); `thread_done` reads `owner_tid` off its own box and calls the split-out `unsafe fn thread_done_one` for the primary box and for each extra one (three sites plus the signature). Nothing new is dereferenced that the single function did not; each carries its SAFETY line | 2026-09-16 | | `segment.rs` | 45 | Segment/page metadata addressing: the mask trick (`segment_of`), `page_of`'s contract-based indexing — **the bound is now PROVED for every in-segment offset by `proofs.rs` (Kani), not merely asserted** — span tiling, purge/recommit. The baseline had drifted to 36 without a row edit; recorded now. **+2 on 2026-10-04 (`docs/plans/recommit-failure-ignored.md`):** `restore_for_reuse` — an `unsafe fn` and its block — is the guard-lift and re-commit `segment_free` and `huge_free` each did inline, moved into one function so both can act on its answer: a segment whose memory the OS will not make accessible and committed again is released whole or retired in place, never recycled. It touches the same range the inline code did; `segment_free`'s own block became the expression that calls it (no net change). `span_recommit` now returns whether the span is backed (same block, same call). **+5 the same day, all `#[cfg(test)]`:** `a_failed_recommit_is_a_failed_span_not_an_unbacked_one` writes and frees the blocks it allocated (four blocks) and frees them at the end (one); each carries its SAFETY line. **+2 on 2026-10-07 (`docs/opps.md` vein census, vein 1):** `scrub_recycled` — an `unsafe fn` and its block — replaces the whole-header `write_bytes` that `segment_alloc` and `huge_alloc` did on every recycled chunk: it zeroes the 2 KB owner table (all of it for a Normal segment, entry 0 for a Huge one, which overwrites the rest) and the first `slots` page slots, inside the same header the old call covered. The other slots of a Normal segment are zeroed where the bump cursor first carves them (inside `span_alloc`'s existing block); a Huge segment reads only slots 0 and 1, and `wait_no_remote_in_flight` now scans slot 1 alone there. Nothing new is dereferenced | 2026-10-07 | diff --git a/crates/rusty_alloc/src/alloc.rs b/crates/rusty_alloc/src/alloc.rs index 65d42d7..ccabd53 100644 --- a/crates/rusty_alloc/src/alloc.rs +++ b/crates/rusty_alloc/src/alloc.rs @@ -359,11 +359,74 @@ unsafe fn malloc_slow(hb: *mut init::HeapBox, size: usize) -> *mut u8 { // only where a workload leaves populated free lists behind // (`mixed`, and `rptest` through `Heap::malloc`). A fast path is worth // its cost only where the thing it looks for is actually there. - return unsafe { (*(*hb).heap.get()).malloc_generic(size).0 }; + // + // The deferred-free hook check stays in TAIL position too: a due hook + // jumps to the cold `malloc_slow_deferred`, which fires it and then + // allocates. Calling `fire_deferred_if_due` here instead put a call in + // this function and gave it a frame: big 91 -> 100, mixed +6.6 Ir/op. + // SAFETY: as above; the flag read forms no reference. + unsafe { + let h = (*hb).heap.get(); + if (*h).deferred_due { + return malloc_slow_deferred(h, size); + } + return (*h).malloc_generic(size).0; + } } malloc_first(size) } +/// [`malloc_slow`] when the heartbeat marked the deferred-free hook due: fire +/// it while no `&mut Heap` exists, then allocate (see +/// [`fire_deferred_if_due`]). +/// +/// # Safety +/// `h` is the calling thread's live, initialised heap. +#[cold] +#[inline(never)] +unsafe fn malloc_slow_deferred(h: *mut crate::heap::Heap, size: usize) -> *mut u8 { + // SAFETY: per contract; the hook runs before `&mut Heap` is formed. + unsafe { + fire_deferred_if_due(h); + (*h).malloc_generic(size).0 + } +} + +/// Fire the deferred-free hook if the generic path's heartbeat marked it due +/// ([`crate::heap::Heap::deferred_due`]). Called at every allocation entry +/// BEFORE it forms `&mut Heap`, which is the point: run inside the heartbeat, +/// a hook that allocates re-entered the heap while `malloc_generic`'s +/// `&mut self` was protected — undefined behaviour under Stacked Borrows +/// (Miri, `oh_f01_deferred_free_hook_that_mallocs_is_contained`, red on main +/// since 2.2.3). Upstream calls the hook at the start of the generic path, +/// with no aliasing rules to break; here it runs at the start of the NEXT +/// allocation entry, which the hook's contract ("called occasionally, with a +/// heartbeat") allows. +/// +/// # Safety +/// `h` is a live heap owned by the calling thread, and no reference to it is +/// live. +#[inline(always)] +unsafe fn fire_deferred_if_due(h: *mut crate::heap::Heap) { + // SAFETY: a raw field read and write on the caller's own heap; no + // reference to the heap is formed, so the hook may re-enter it. + unsafe { + if (*h).deferred_due { + (*h).deferred_due = false; + fire_deferred_now(); + } + } +} + +/// The hook call itself, out of line and cold: it is an indirect call that is +/// almost never reached, and inlined it would cost every entry's register +/// allocation. +#[cold] +#[inline(never)] +fn fire_deferred_now() { + crate::options::deferred_free(false); +} + /// A fresh thread's very first allocation: create the heap, then allocate. /// Split from [`malloc_slow`] so its register needs (keeping `size` alive /// across `init_thread_heap`) are paid once per THREAD, not once per slow @@ -437,7 +500,11 @@ unsafe fn zalloc_slow(hb: *mut init::HeapBox, size: usize) -> *mut u8 { return zalloc_first(size); } // SAFETY: a non-sentinel box is this thread's live, initialised box. - unsafe { (*(*hb).heap.get()).zalloc(size) } + unsafe { + let h = (*hb).heap.get(); + fire_deferred_if_due(h); + (*h).zalloc(size) + } } /// A fresh thread's first zeroing allocation: create the heap, then allocate. @@ -654,7 +721,10 @@ fn malloc_aligned_first(size: usize, align: usize, offset: usize) -> *mut u8 { return ptr::null_mut(); // heap creation failed: OOM ⇒ null } // SAFETY: own heap. - unsafe { (*h).malloc_aligned_at(size, align, offset).0 } + unsafe { + fire_deferred_if_due(h); + (*h).malloc_aligned_at(size, align, offset).0 + } } /// `mi_zalloc_aligned`. @@ -679,7 +749,9 @@ pub fn zalloc_aligned_at(size: usize, align: usize, offset: usize) -> *mut u8 { // SAFETY: a non-sentinel box is this thread's live, initialised box; // zero_block contract (zeroes [p, p+usable)). unsafe { - let (p, is_zero) = (*(*hb).heap.get()).malloc_aligned_at(size, align, offset); + let h = (*hb).heap.get(); + fire_deferred_if_due(h); + let (p, is_zero) = (*h).malloc_aligned_at(size, align, offset); if !p.is_null() { zero_block(p, is_zero); } @@ -697,6 +769,7 @@ fn zalloc_aligned_first(size: usize, align: usize, offset: usize) -> *mut u8 { } // SAFETY: own heap; zero_block contract. unsafe { + fire_deferred_if_due(h); let (p, is_zero) = (*h).malloc_aligned_at(size, align, offset); if !p.is_null() { zero_block(p, is_zero); @@ -1570,7 +1643,11 @@ unsafe fn heap_of(hb: *mut init::HeapBox) -> *mut Heap { /// `hb` live and owned by the calling thread. pub unsafe fn heap_malloc(hb: *mut init::HeapBox, size: usize) -> *mut u8 { // SAFETY: forwarded contract. - unsafe { (*heap_of(hb)).malloc(size).0 } + unsafe { + let h = heap_of(hb); + fire_deferred_if_due(h); + (*h).malloc(size).0 + } } /// `mi_heap_zalloc`. @@ -1580,7 +1657,9 @@ pub unsafe fn heap_malloc(hb: *mut init::HeapBox, size: usize) -> *mut u8 { pub unsafe fn heap_zalloc(hb: *mut init::HeapBox, size: usize) -> *mut u8 { // SAFETY: forwarded contract. unsafe { - let (p, is_zero) = (*heap_of(hb)).malloc(size); + let h = heap_of(hb); + fire_deferred_if_due(h); + let (p, is_zero) = (*h).malloc(size); if !p.is_null() { zero_block(p, is_zero); } @@ -1599,7 +1678,11 @@ pub unsafe fn heap_malloc_aligned_at( offset: usize, ) -> *mut u8 { // SAFETY: forwarded contract. - unsafe { (*heap_of(hb)).malloc_aligned_at(size, align, offset).0 } + unsafe { + let h = heap_of(hb); + fire_deferred_if_due(h); + (*h).malloc_aligned_at(size, align, offset).0 + } } /// `mi_heap_zalloc_aligned_at`. @@ -1614,7 +1697,9 @@ pub unsafe fn heap_zalloc_aligned_at( ) -> *mut u8 { // SAFETY: forwarded contract. unsafe { - let (p, is_zero) = (*heap_of(hb)).malloc_aligned_at(size, align, offset); + let h = heap_of(hb); + fire_deferred_if_due(h); + let (p, is_zero) = (*h).malloc_aligned_at(size, align, offset); if !p.is_null() { zero_block(p, is_zero); } diff --git a/crates/rusty_alloc/src/heap.rs b/crates/rusty_alloc/src/heap.rs index 1a8a0a8..b0f8d3e 100644 --- a/crates/rusty_alloc/src/heap.rs +++ b/crates/rusty_alloc/src/heap.rs @@ -205,6 +205,13 @@ pub struct Heap { /// conservative direction (it falls back to the behaviour that has always /// shipped). pub saw_remote_free: bool, + /// The heartbeat found a deferred-free hook registered: fire it at the + /// next allocation entry, BEFORE that entry forms `&mut Heap`. Firing it + /// from the heartbeat itself ran the user's hook while `&mut self` was + /// live, and a hook that allocates re-entered this heap through a raw + /// pointer under that protected borrow: undefined behaviour (Miri, + /// Stacked Borrows, `oh_f01_deferred_free_hook_that_mallocs_is_contained`). + pub deferred_due: bool, /// Trips of the generic path left before the next automatic collect. /// /// `mi_option_generic_collect`. Counts DOWN so the hot check is a compare @@ -267,6 +274,7 @@ impl Heap { tag: 0, rng: crate::random::Random::new(), saw_remote_free: false, + deferred_due: false, generic_countdown: 0, guarded_rate: 0, guarded_count: 0, @@ -670,10 +678,15 @@ impl Heap { } // Heartbeat: process cross-thread delayed frees at slow-path cadence // (this is what un-parks full pages whose blocks died remotely), and - // fire the registered deferred-free hook (mi_register_deferred_free). + // mark the registered deferred-free hook (mi_register_deferred_free) + // DUE. It is not called here: `&mut self` is live, and a hook that + // allocates would re-enter this heap under it. `alloc` fires it at + // the next entry, before any `&mut Heap` exists (`deferred_due`). // SAFETY: we are the owner thread. unsafe { self.process_delayed() }; - crate::options::deferred_free(false); + if crate::options::deferred_registered() { + self.deferred_due = true; + } // Periodic collect (`mi_option_generic_collect`, default 10,000). // // Upstream runs an UNFORCED collect every N trips of this path. Ours diff --git a/crates/rusty_alloc/src/options.rs b/crates/rusty_alloc/src/options.rs index 00cd5aa..f899111 100644 --- a/crates/rusty_alloc/src/options.rs +++ b/crates/rusty_alloc/src/options.rs @@ -913,7 +913,17 @@ fn render_error(buf: &mut [u8; 32], err: i32) -> &str { ) } -/// Fire the deferred-free hook (called from the allocation heartbeat). +/// Whether a deferred-free hook is registered: the function-pointer peek of +/// [`deferred_free`], without calling anything. The generic path's heartbeat +/// uses it to mark a hook call DUE; the call itself happens at the +/// raw-pointer boundary in `alloc`, where no `&mut Heap` is alive. +#[inline] +pub fn deferred_registered() -> bool { + !DEFERRED_FUN.load_fun().is_null() +} + +/// Fire the deferred-free hook. Called from `alloc`'s allocation entries +/// when the heartbeat has marked it due, never with a `&mut Heap` live. pub fn deferred_free(force: bool) { // Peek at the FUNCTION pointer alone first. `mi_register_deferred_free` is // unregistered in nearly every process, and this runs on every slow-path diff --git a/tools/unsafe-baseline.txt b/tools/unsafe-baseline.txt index fc46944..02a3289 100644 --- a/tools/unsafe-baseline.txt +++ b/tools/unsafe-baseline.txt @@ -1,4 +1,4 @@ - 106 crates/rusty_alloc/src/alloc.rs + 110 crates/rusty_alloc/src/alloc.rs 15 crates/rusty_alloc/src/arena.rs 80 crates/rusty_alloc/src/heap.rs 42 crates/rusty_alloc/src/init.rs