From 279209c5b573cc7ac202d94d57638be1c1d3a6c2 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 7 Oct 2026 23:17:52 -0700 Subject: [PATCH 1/2] perf: vein census 3 -- nine deterministic wins from real-program profiles Census by function on real programs (perl, sqlite, lua, python, Endless Sky start-up) and the Rust GlobalAlloc workloads; docs/opps.md "Vein census 3". Allocator Ir vs 2.2.5, outputs identical: Endless Sky -6.85 %, lua -2.23 %, perl -1.64 %, sqlite -0.11 %; Rust overaligned -4.24 %, threads -1.53 %, maps -0.36 %. Peak RSS within +-0.4 %. 1. page_extend carves in geometric batches (at least what the page holds): the slow-path trips, not the 2.75 Ir/block linking, were the cost. ES -4.50 %. A fixed 8 KiB batch cost thread life +10 % (refuted). 2. C exports test the owner before the page flags, so a cross-thread free into a FULL page goes straight to remote_free (ES -515 k, xthread -2.9 %). Rust's dealloc keeps the 2.2.5 order (free_inline_flags_first): the bigger remote arm stopped __rust_dealloc inlining into drop glue (trees +2.88 %). Trade: a local free into a slow page +2 Ir. 3. GlobalAlloc::alloc: only the natural-alignment arm out of line (it was a second inlined malloc copy). maps -0.33 %, overaligned -1.81 %. 4. SMALL_BIN table for the generic path's bin, by the word count it already computes. ES -0.70 % and more. 5. malloc_or_with::: operator new's cold arm frame 12 -> 7. ES -419 k. 6. realloc_live hands its resolved page to the free (lua -60 k). 7. An 8-slot heap-box cache (upstream TD_CACHE_SIZE): no mmap/munmap pair per thread. Thread life -1.1 %, minor faults -15 %. 8. #[inline] usable_size: over-aligned Rust realloc -2.06 %. 9. MSVC + panic=unwind: sub/setle into an output operand, +2 over the goto where 2.2.5's plain decrement was +3. Still 0 callbr there. 10. Thread-exit collect: refuted as inherent (the cross-thread drain). Gates: workspace --all-features 177/177, linux-gates 42/0, embedded 15/15, clippy (Windows + Linux), fmt, semgrep, wasm self-test + size, gate self-test, miri gate + openheimer 37/37, mimalloc-bench 19/19, churn 5/5, real-world outputs identical across ra/mi/sys, alloc-eval 8 traces, rusty_sloth built with crates.io identity (--release --features cuda). Unsafe census 981 -> 994 (UNSAFE.md). Co-Authored-By: Claude Opus 5.5 --- bench/rust-globalalloc/Cargo.lock | 4 +- crates/rusty_alloc/CHANGELOG.md | 34 + crates/rusty_alloc/UNSAFE.md | 2 +- crates/rusty_alloc/src/alloc.rs | 279 ++- crates/rusty_alloc/src/bins.rs | 35 +- crates/rusty_alloc/src/heap.rs | 2 +- crates/rusty_alloc/src/init.rs | 2547 ++++++++++++++------------- crates/rusty_alloc/src/page.rs | 15 +- crates/rusty_alloc_api/CHANGELOG.md | 8 + crates/rusty_alloc_api/src/lib.rs | 25 +- crates/rusty_alloc_ffi/src/lib.rs | 12 +- docs/opps.md | 80 + tools/unsafe-baseline.txt | 2 +- 13 files changed, 1743 insertions(+), 1302 deletions(-) diff --git a/bench/rust-globalalloc/Cargo.lock b/bench/rust-globalalloc/Cargo.lock index 017717b..8c832fc 100644 --- a/bench/rust-globalalloc/Cargo.lock +++ b/bench/rust-globalalloc/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "rusty_alloc" -version = "2.2.3" +version = "2.2.5" dependencies = [ "libc", "portable-atomic", @@ -32,7 +32,7 @@ dependencies = [ [[package]] name = "rusty_alloc-api" -version = "2.2.3" +version = "2.2.5" dependencies = [ "rusty_alloc", ] diff --git a/crates/rusty_alloc/CHANGELOG.md b/crates/rusty_alloc/CHANGELOG.md index 9c6ff40..f623675 100644 --- a/crates/rusty_alloc/CHANGELOG.md +++ b/crates/rusty_alloc/CHANGELOG.md @@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Performance + +Measured against 2.2.5 with callgrind, allocator instructions, outputs +identical: Endless Sky start-up -6.85 %, lua -2.23 %, perl -1.64 %, sqlite +-0.11 %; Rust `GlobalAlloc` workloads up to -4.24 % whole-program. Peak RSS +within +-0.4 %. Details: `docs/opps.md`, "Vein census 3". + +- **Pages carve in growing batches.** A page's first extension is still one + 4 KiB OS page; each later one carves as many blocks as the page already + holds. Each extension costs a slow-path trip, and those trips, not the + linking, were the cost. Thread start-up is unchanged. +- **A cross-thread free into a full page goes straight to its owner** (the + `free`/`operator delete` exports test the owner first). Rust's `dealloc` + keeps the 2.2.5 order, which keeps `__rust_dealloc` inlinable into drop + glue. Trade-off: a same-thread free of a large or huge block costs 2 more + instructions (about 1.4 % of that path). +- **The slow path looks up a small size's bin from a table.** +- **`operator new`'s slow path has a smaller frame** (`malloc_or_with`, a + typed OOM handler). +- **A moving `realloc` resolves the block's page once, not twice.** +- **Exited threads' heap boxes are reused**, saving an `mmap`/`munmap` pair + and a page fault per thread. +- **Rust: the natural-alignment arm of `GlobalAlloc::alloc` is out of line, + and `usable_size` is inlinable.** +- **MSVC builds with `panic = "unwind"`** take back one of the three + instructions per free that 2.2.5's fix cost them. + +### Added + +- `alloc::malloc_or_with` and the `alloc::OnOom` trait: `malloc_or` with the + OOM handler as a type. +- `alloc::free_inline_flags_first`: `free_inline`'s body in the 2.2.5 test + order, for Rust's `GlobalAlloc::dealloc`. + ## [2.2.5](https://github.com/Remade-With-Rust/rusty_alloc/compare/rusty_alloc-v2.2.4...rusty_alloc-v2.2.5) - 2026-10-08 ### Fixed diff --git a/crates/rusty_alloc/UNSAFE.md b/crates/rusty_alloc/UNSAFE.md index 89c7134..fbb39f0 100644 --- a/crates/rusty_alloc/UNSAFE.md +++ b/crates/rusty_alloc/UNSAFE.md @@ -20,7 +20,7 @@ unsafe in DEPENDENCIES, and ours are `libc` plus bindings-only `windows-sys`. | Module | Count | What the `unsafe` is for | Last audit | |---|---:|---|---| -| `alloc.rs` | 110 | The public entry points: raw-pointer reads on the malloc fast path (must never form `&mut` on the shared empty-heap sentinel), pointer-derived metadata on the free path (`segment_of`/`page_of`), block-content copies in the realloc family. **+15 on 2026-08-22**: the free path's two `asm!` sites — the memory-destination `used--` whose flags drive the retire branch (its `label` block is a separate item and carries its own `unsafe`), and the fused `cmp {tid}, fs:0` in both `free_inline` and `free_general` — plus `malloc_or`/`malloc_or_slow`, which give `operator new` a fast path whose miss is a tail call. Each asm reads or writes exactly one field it already had a valid pointer to; none widens what the surrounding code could already touch. **+2 on 2026-09-24 (instruction-count campaign, LEDGER):** `realloc` split into an inline null test and `unsafe fn realloc_live(p: NonNull, ..)` — one signature, and the `NonNull::new_unchecked` sits inside the block that already called it, on a pointer the line above proved non-null. The body is the old one unchanged; nothing new is dereferenced. **+10 on 2026-09-24 (round two, LEDGER):** (a) `realloc_move`, the moving arm shared by the growth and the shrink — one `unsafe fn`, its body block, and the two call-site blocks: the old inline move code, now reached with the copy length each arm already holds; (b) `zalloc` in `malloc`'s raw-read shape — its fast-path block reads through `hb`, which may be the sentinel, exactly as `malloc` does (no `&mut`, no write unless a block was popped, which proves the heap is real), and the cold `unsafe fn zalloc_slow` that performs the sentinel test the fast path dropped (signature + block); (c) `free_local_owned` / `free_local_no_xheap`, `owner_heap` split so its heap-creating fallback is a tail call — two signatures and their two blocks, the same loads and the same `free_local_at` as before; (d) `malloc_aligned_pow2`, an `unsafe fn` only because its caller must have proven the alignment a power of two (a violation is not memory-unsafe — the bound test still refuses a mask at or above half a segment). Nothing new is dereferenced anywhere; each carries its SAFETY line. The baseline had drifted to 106 without a row edit; recorded now. **+4 on 2026-10-07 (the Miri Stacked Borrows failure on `main`):** the deferred-free hook used to fire inside `Heap::malloc_generic`, where `&mut self` is protected, so a hook that allocates re-entered the heap through a raw pointer under that borrow. It now fires at the allocation entries before any `&mut Heap` exists: `fire_deferred_if_due` (an `unsafe fn` and its block: a raw read and write of the heap's own `deferred_due` flag, then the hook) and `malloc_slow_deferred` (an `unsafe fn` and its block: the same, then the `malloc_generic` call `malloc_slow` already made). The nine entry blocks that call it existed before; nothing new is dereferenced | 2026-10-07 | +| `alloc.rs` | 123 | The public entry points: raw-pointer reads on the malloc fast path (must never form `&mut` on the shared empty-heap sentinel), pointer-derived metadata on the free path (`segment_of`/`page_of`), block-content copies in the realloc family. **+15 on 2026-08-22**: the free path's two `asm!` sites — the memory-destination `used--` whose flags drive the retire branch (its `label` block is a separate item and carries its own `unsafe`), and the fused `cmp {tid}, fs:0` in both `free_inline` and `free_general` — plus `malloc_or`/`malloc_or_slow`, which give `operator new` a fast path whose miss is a tail call. Each asm reads or writes exactly one field it already had a valid pointer to; none widens what the surrounding code could already touch. **+2 on 2026-09-24 (instruction-count campaign, LEDGER):** `realloc` split into an inline null test and `unsafe fn realloc_live(p: NonNull, ..)` — one signature, and the `NonNull::new_unchecked` sits inside the block that already called it, on a pointer the line above proved non-null. The body is the old one unchanged; nothing new is dereferenced. **+10 on 2026-09-24 (round two, LEDGER):** (a) `realloc_move`, the moving arm shared by the growth and the shrink — one `unsafe fn`, its body block, and the two call-site blocks: the old inline move code, now reached with the copy length each arm already holds; (b) `zalloc` in `malloc`'s raw-read shape — its fast-path block reads through `hb`, which may be the sentinel, exactly as `malloc` does (no `&mut`, no write unless a block was popped, which proves the heap is real), and the cold `unsafe fn zalloc_slow` that performs the sentinel test the fast path dropped (signature + block); (c) `free_local_owned` / `free_local_no_xheap`, `owner_heap` split so its heap-creating fallback is a tail call — two signatures and their two blocks, the same loads and the same `free_local_at` as before; (d) `malloc_aligned_pow2`, an `unsafe fn` only because its caller must have proven the alignment a power of two (a violation is not memory-unsafe — the bound test still refuses a mask at or above half a segment). Nothing new is dereferenced anywhere; each carries its SAFETY line. The baseline had drifted to 106 without a row edit; recorded now. **+4 on 2026-10-07 (the Miri Stacked Borrows failure on `main`):** the deferred-free hook used to fire inside `Heap::malloc_generic`, where `&mut self` is protected, so a hook that allocates re-entered the heap through a raw pointer under that borrow. It now fires at the allocation entries before any `&mut Heap` exists: `fire_deferred_if_due` (an `unsafe fn` and its block: a raw read and write of the heap's own `deferred_due` flag, then the hook) and `malloc_slow_deferred` (an `unsafe fn` and its block: the same, then the `malloc_generic` call `malloc_slow` already made). The nine entry blocks that call it existed before; nothing new is dereferenced **+13 on 2026-10-08 (vein census 3, `docs/opps.md`):** every one is a split or a re-shape of an existing path, and nothing new is dereferenced. `free_inline_impl` + `free_inline_flags_first` + `free_body` (the free fast path split by test order and by whether the page is already known: three `unsafe fn`s and the blocks that forward the contract); a second owner-test `asm!` arm whose label calls `remote_free` exactly as before; `remote_free_flagged` (an `unsafe fn`, its `unalign` block and its `remote_free` block: the same unalign `free_general` did for a remote aligned block); `realloc_live`'s page resolution (the `segment_of`/`page_of`/flags read `usable_size` did) and `realloc_move_at` replacing `realloc_move` (same allocate-copy-free, freeing through the page already resolved); `malloc_or_with`/`malloc_or_slow_with` (the `malloc_or` fast path and cold arm with a typed handler). The MSVC-unwind `sub`/`setle` arm sits inside the existing fast-path block | 2026-10-08 | | `heap.rs` | 80 | Owner-thread page/queue manipulation under raw pointers (no two `&mut Page` may coexist), the aligned-allocation peek, span carving. **+3 on 2026-08-19**: `try_unlink_huge_segment` split out of `remove_huge_segment`. **+15 on 2026-08-22**: `malloc_generic` split into a small entry plus `malloc_generic_walk`, with `grow_front`, `try_guarded` and `drain_delayed` as cold arms — each split adds an `unsafe fn` signature and its block while dereferencing nothing the single function did not — and the immortal `EMPTY_DELAYED` sentinel that lets the heartbeat read its list without a null test. **+2 on 2026-09-07 (P4e, §2.15 of `docs/plans/small-metal.md`)**: the reclamation fixes — one `unsafe` for the periodic `generic_collect` sweep and one for the reclaim-and-retry that runs before `malloc_generic` returns null. Both call `collect_inner`, which allocates nothing and touches only this heap’s own pages on the owner thread; each carries its SAFETY line. `malloc_generic` itself became a safe wrapper over the renamed `malloc_generic_once`, so the split added no signature. **+2 on 2026-09-08:** the medium collect-and-retry ahead of the heartbeat -- one `unsafe` around `page_collect` + `page_pop` on the bin queue front, one reading `free_is_zero` off the page just popped. Both are the SAME operations `malloc_generic_walk` performs on the same page a few lines later, on the owner thread under the heap lock: the block moves earlier, nothing new is dereferenced. Each carries its SAFETY line. **+2 on 2026-09-16 (Openheimer):** the two refuse arms in `malloc_aligned_at_slow` when `bins::aligned_at_from` reports that `block + offset + align` would wrap for a caller-chosen offset — each is one `free_local` of the block just allocated on this thread, which has not escaped. **+1 on 2026-09-24 (instruction-count campaign, LEDGER):** the medium collect-and-retry's `page_collect` + `page_pop` block became two blocks so the `saw_remote_free` latch is stored between them — same two calls on the same page, owner thread, in the same order; nothing new is dereferenced. **+1 on 2026-09-24 (round two):** a medium miss grows the front page directly — one block around the `capacity < reserved` test and the `grow_front` call, on the same page the block above just collected, which is `grow_front`'s whole contract **+2 on 2026-09-25 (CURIOSITY ROUND FOUR):** the medium arm pops the front page BEFORE collecting it — one `page_pop` block and the `free_is_zero` read beside it, the same two operations the collect-then-pop below already performs on the same owner-thread page. **+1 on 2026-10-07 (`docs/opps.md` vein census, vein 2):** `collect_inner` hands the per-heap cached large span (`Heap::large_cache`) back with `retire_span` — one block around `segment_of` on the cached page's own address and the same `retire_span` a large free performed before the cache existed, on the owner thread. **+2 the same day:** `free_large_span` — an `unsafe fn` and its block — is the large-span arm of `free_local_at` moved into its own function (double-free check, cache, free-run flag); the same operations on the same page, owner thread | 2026-10-07 | | `init.rs` | 42 | Thread/heap lifecycle: the initial-exec TLS slot (`global_asm!` + fs-relative asm reads), thread-pointer register reads (`fs:0`/`gs:0x30`/`tpidrro_el0`), heap-box creation/teardown, the abandonment path run inside platform TLS destructors. **+1 on 2026-09-09 (`firmware-what-is-left.md` §7.3): an ATTRIBUTE, not an operation** — `unsafe(link_section = ".rodata.…")` on `EMPTY_HEAP_BOX`, bare metal only, so the never-written sentinel lives in flash instead of costing 1,752 bytes of RAM. The token is counted because the census is a text search; the contract it rests on (raw reads only, no page ever stores its address) is the one the `Sync` impl below already carries, and a write would now fault against the flash cache instead of silently landing. `create_heap`'s template copy from it is a `ptr::read` inside the block that already existed. **+5 on 2026-09-16 (Openheimer, OH-rusty_alloc-13):** thread exit now abandons EVERY heap the dying thread still owns, not only the one in `done_slot` — a first-class heap that was never `heap_delete`d, or one installed with `set_default_heap`, used to stay DELAYED under a dead owner forever. `take_heap_owned_by` unlinks one such box under `HEAPS_LOCK` (one block: the registry walk); `thread_done` reads `owner_tid` off its own box and calls the split-out `unsafe fn thread_done_one` for the primary box and for each extra one (three sites plus the signature). Nothing new is dereferenced that the single function did not; each carries its SAFETY line | 2026-09-16 | | `segment.rs` | 45 | Segment/page metadata addressing: the mask trick (`segment_of`), `page_of`'s contract-based indexing — **the bound is now PROVED for every in-segment offset by `proofs.rs` (Kani), not merely asserted** — span tiling, purge/recommit. The baseline had drifted to 36 without a row edit; recorded now. **+2 on 2026-10-04 (`docs/plans/recommit-failure-ignored.md`):** `restore_for_reuse` — an `unsafe fn` and its block — is the guard-lift and re-commit `segment_free` and `huge_free` each did inline, moved into one function so both can act on its answer: a segment whose memory the OS will not make accessible and committed again is released whole or retired in place, never recycled. It touches the same range the inline code did; `segment_free`'s own block became the expression that calls it (no net change). `span_recommit` now returns whether the span is backed (same block, same call). **+5 the same day, all `#[cfg(test)]`:** `a_failed_recommit_is_a_failed_span_not_an_unbacked_one` writes and frees the blocks it allocated (four blocks) and frees them at the end (one); each carries its SAFETY line. **+2 on 2026-10-07 (`docs/opps.md` vein census, vein 1):** `scrub_recycled` — an `unsafe fn` and its block — replaces the whole-header `write_bytes` that `segment_alloc` and `huge_alloc` did on every recycled chunk: it zeroes the 2 KB owner table (all of it for a Normal segment, entry 0 for a Huge one, which overwrites the rest) and the first `slots` page slots, inside the same header the old call covered. The other slots of a Normal segment are zeroed where the bump cursor first carves them (inside `span_alloc`'s existing block); a Huge segment reads only slots 0 and 1, and `wait_no_remote_in_flight` now scans slot 1 alone there. Nothing new is dereferenced | 2026-10-07 | diff --git a/crates/rusty_alloc/src/alloc.rs b/crates/rusty_alloc/src/alloc.rs index ca61d00..c681f9c 100644 --- a/crates/rusty_alloc/src/alloc.rs +++ b/crates/rusty_alloc/src/alloc.rs @@ -304,6 +304,53 @@ pub fn malloc_or(size: usize, on_oom: fn(usize) -> *mut u8) -> *mut u8 { } } +/// A compile-time OOM handler for [`malloc_or_with`]. +pub trait OnOom { + /// Called once the ordinary slow path has returned null for `size`. + fn on_oom(size: usize) -> *mut u8; +} + +/// [`malloc_or`] with the handler as a TYPE rather than a `fn` pointer. +/// +/// The cold arm is out of line, so inside it a `fn` pointer is a runtime +/// value that has to survive the slow-path call next to `size`: two +/// callee-saved registers and an alignment slot, a 12-instruction frame on +/// every slow `operator new` (Endless Sky: 70,115 calls, 841 k Ir). With a +/// type the handler call is direct and only `size` is live. +#[inline] +pub fn malloc_or_with(size: usize) -> *mut u8 { + let hb = init::heap_box_fast(); + // SAFETY: as [`malloc_or`]. + unsafe { + if size <= SMALL_SIZE_MAX { + let w = crate::types::wsize_from_size(size); + let h = (*hb).heap.get(); + let p = (*h).direct[w]; + let b = crate::page::page_pop(p); + if !b.is_null() { + #[cfg(debug_assertions)] + { + (*h).stats.allocs += 1; + } + return b; + } + } + malloc_or_slow_with::(hb, size) + } +} + +/// The cold arm of [`malloc_or_with`]. +/// +/// # Safety +/// As [`malloc_slow`]. +#[cold] +#[inline(never)] +unsafe fn malloc_or_slow_with(hb: *mut init::HeapBox, size: usize) -> *mut u8 { + // SAFETY: forwarded contract. + let p = unsafe { malloc_slow(hb, size) }; + if p.is_null() { H::on_oom(size) } else { p } +} + /// The cold arm of [`malloc_or`]: the ordinary slow path, then the caller's /// OOM handler if even that could not serve. /// @@ -980,6 +1027,37 @@ pub unsafe fn free(p: *mut u8) { /// As [`free`]. #[inline(always)] pub unsafe fn free_inline(p: *mut u8) { + // SAFETY: forwarded contract. + unsafe { free_inline_impl::(p) } +} + +/// [`free_inline`] with the 2.2.5 test order (page flags, THEN owner), for +/// `rusty_alloc_api`'s `GlobalAlloc::dealloc`. +/// +/// SPLIT BODY, TWO SYMBOLS, because the two kinds of caller want different +/// shapes. The owner-first order sends a cross-thread free into a FULL page +/// straight to `remote_free` (Endless Sky allocator Ir -514,197, opscan +/// `xthread` 77.09 -> 74.82), but its remote arm is a little more code, and +/// inlined into Rust's `__rust_dealloc` that tipped LLVM out of inlining the +/// shim into drop glue: `trees` +2.88 % whole-program (`__rust_dealloc` 46 -> +/// 4,099,351 Ir self), against `maps` -1.19 %. A Rust drop is rarely +/// cross-thread, so Rust keeps the smaller body. +/// +/// # Safety +/// As [`free`]. +#[inline(always)] +pub unsafe fn free_inline_flags_first(p: *mut u8) { + // SAFETY: forwarded contract. + unsafe { free_inline_impl::(p) } +} + +/// The body of [`free_inline`] and [`free_inline_flags_first`]; +/// `OWNER_FIRST` picks the test order and folds at compile time. +/// +/// # Safety +/// As [`free`]. +#[inline(always)] +unsafe fn free_inline_impl(p: *mut u8) { // REFUTED TWICE (2026-08-22): folding the null test into the segment mask, the // way mimalloc does (`lea -0x1(%rdi),%rsi; and mask,%rsi; jle` — a block is // never at offset 0, so masking `p - 1` picks the same segment, and NULL @@ -1004,6 +1082,25 @@ pub unsafe fn free_inline(p: *mut u8) { } debug_foreign_pointer_guard(p); let seg = segment_of(p); + // SAFETY: forwarded contract; `seg` is `p`'s segment. + unsafe { free_body::(p, seg, ptr::null_mut()) } +} + +/// The free fast path from the segment on. `KNOWN_PAGE` callers pass the +/// page they already resolved (`realloc_live`, which resolved it for its +/// usable size); the others pass null and the page is resolved here, in the +/// same place as before, AFTER the owner-id load (see the note on that order +/// below). Both are compile-time choices. +/// +/// # Safety +/// As [`free`], `seg == segment_of(p)`, and with `KNOWN_PAGE`, +/// `known_pg == page_of(seg, p)`. +#[inline(always)] +unsafe fn free_body( + p: *mut u8, + seg: *mut Segment, + known_pg: *mut Page, +) { // SAFETY: p is ours per the contract → seg is a live segment header. The // OWNING heap is recovered from the page's xheap back-pointer (container- // of over the HeapBox's offset-0 delayed list) — correct even when the @@ -1043,18 +1140,33 @@ pub unsafe fn free_inline(p: *mut u8) { // normal segment, single-block span, interior (aligned-at) pointer. // page_of works for both kinds: a huge segment's interior slices all // offset back to slot 1. - let pg = page_of(seg, p); + let pg = if KNOWN_PAGE { + known_pg + } else { + page_of(seg, p) + }; let flags = (*pg).flags.load(Ordering::Relaxed); // ONE test decides the whole shape of the free (upstream's // `page->flags.full_aligned == 0`). A clear byte means: binned page in // a Normal segment, queued, exact pointer — so the general path's // segment-kind match, bin compare, full-queue re-test and unalign are // all provably unnecessary and are skipped rather than re-derived. - if flags & pflags::SLOW_FREE == 0 { - // SAFETY: reads the thread pointer at `fs:0` and compares it with - // the segment's owner id — the same test as `owner_tid == - // init::thread_id()`, with the load folded into the compare's - // memory operand. Reads only; no stack. + // The OWNER test comes before the flags test, so a cross-thread free + // leaves here whatever its page's flags say. Flags-first sent every + // remote free into a FULL page (the common cross-thread case: the + // owner filled the page, another thread frees into it) through + // `free_general`'s frame and second flags load just to reach the same + // `remote_free` (Endless Sky: 173 k of 175 k general frees). A remote + // free needs the flags only to unalign an interior pointer. Measured: + // Endless Sky allocator Ir -514,197 (-0.90 %), opscan xthread 77.09 -> + // 74.82, every local op unchanged. Inlining the unalign into this arm + // instead cost EVERY free +3 Ir (small 52.26 -> 55.26), and a single + // cold call for all remote frees kept only -43,649 of it. + // SAFETY: reads the thread pointer at `fs:0` and compares it with the + // segment's owner id — the same test as `owner_tid == + // init::thread_id()`, with the load folded into the compare's memory + // operand. Reads only; no stack. + if OWNER_FIRST { #[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] core::arch::asm!( "cmp {tid}, fs:0", @@ -1063,7 +1175,12 @@ pub unsafe fn free_inline(p: *mut u8) { remote = label { // SAFETY: `pg` is the live page this free resolved; a // non-owning thread hands the block to its owner. - unsafe { remote_free(pg, p.cast::()) }; + unsafe { + if flags & pflags::HAS_ALIGNED != 0 { + return remote_free_flagged(pg, p, flags); + } + remote_free(pg, p.cast::()) + }; return; }, options(nostack, readonly), @@ -1073,9 +1190,37 @@ pub unsafe fn free_inline(p: *mut u8) { // ordinary comparison below. #[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))] if !local { + if flags & pflags::HAS_ALIGNED != 0 { + return remote_free_flagged(pg, p, flags); + } remote_free(pg, p.cast::()); return; } + } + if flags & pflags::SLOW_FREE == 0 { + if !OWNER_FIRST { + // The 2.2.5 order: only a clear flags byte reaches the owner + // test here; a remote free into a full page goes through + // `free_general`. + // SAFETY: as the owner-first arm above. + #[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] + core::arch::asm!( + "cmp {tid}, fs:0", + "jne {remote}", + tid = in(reg) owner_tid, + remote = label { + // SAFETY: as above. + unsafe { remote_free(pg, p.cast::()) }; + return; + }, + options(nostack, readonly), + ); + #[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))] + if !local { + remote_free(pg, p.cast::()); + return; + } + } { // Routing the whole free on ONE byte is only safe while that // byte agrees with the fields it summarises. These two check @@ -1198,11 +1343,41 @@ pub unsafe fn free_inline(p: *mut u8) { options(nostack), ); } - #[cfg(not(all( + // MSVC with `panic = "unwind"`: no goto (see the arm above), + // but an `asm!` with an OUTPUT operand is allowed there, and + // `sub` + `setle` is four instructions with the test, where + // the plain decrement below is five (load, dec, store, test, + // branch: LLVM will not emit a memory-destination RMW whose + // result drives a branch). Measured with this arm forced on + // Linux: see the note below the cfg. + #[cfg(all( target_arch = "x86_64", not(miri), - not(all(target_env = "msvc", panic = "unwind")) - )))] + target_env = "msvc", + panic = "unwind" + ))] + { + let le: u8; + // SAFETY: as the goto arm: `pg` is a live page of this + // thread, `USED_OFFSET` is `offset_of!(Page, used)`, and + // the asm touches only that `u32` and uses no stack. + core::arch::asm!( + "sub dword ptr [{pg} + {off}], 1", + "setle {le}", + pg = in(reg) pg, + off = const crate::page::USED_OFFSET, + le = out(reg_byte) le, + options(nostack), + ); + if le != 0 { + // One test, as in the goto arm. + if ((*pg).used as usize | (*pg).next.addr() | (*pg).prev.addr()) == 0 { + return; + } + return retire_or_abort(pg); + } + } + #[cfg(not(all(target_arch = "x86_64", not(miri))))] { let u = (*pg).used.wrapping_sub(1); (*pg).used = u; @@ -1282,6 +1457,27 @@ unsafe fn retire_or_abort(pg: *mut Page) { } } +/// A cross-thread free of a block on a page that holds aligned-at blocks: +/// hand the owner the block itself, not the interior pointer. Cold and out of +/// line, so the remote arm of the free fast path stays a direct +/// `remote_free` for every other block (inlining the unalign there cost every +/// free +3 Ir, local ones included). +/// +/// # Safety +/// `pg` is the live page of `p`, and `flags` its flags byte. +#[cold] +#[inline(never)] +unsafe fn remote_free_flagged(pg: *mut Page, p: *mut u8, flags: u8) { + let b = if flags & pflags::HAS_ALIGNED != 0 && flags & pflags::HUGE_SEGMENT == 0 { + // SAFETY: an aligned-at block on a binned page, per the flags. + unsafe { unalign(pg, p) } + } else { + p + }; + // SAFETY: as `free_general`'s remote arm. + unsafe { remote_free(pg, b.cast::()) } +} + /// Everything a free can be once the flags byte is NOT clear: an interior /// (aligned-at) pointer, a huge segment, an unqueued single-block span, or a /// page parked in the full queue. Counted at 1.57% of frees on `batch_lifo`. @@ -1388,6 +1584,7 @@ unsafe fn free_general(p: *mut u8, seg: *mut Segment, pg: *mut Page, owner_tid: /// /// # Safety /// `p` must be null or a live pointer from this allocator. +#[inline] pub unsafe fn usable_size(p: *const u8) -> usize { if p.is_null() { return 0; @@ -1490,8 +1687,24 @@ pub unsafe fn realloc(p: *mut u8, newsize: usize) -> *mut u8 { #[inline(never)] unsafe fn realloc_live(p: ptr::NonNull, newsize: usize) -> *mut u8 { let p = p.as_ptr(); - // SAFETY: p live per contract. - let usable = unsafe { usable_size(p) }; + // The segment and page are resolved ONCE, here, and handed to the free + // of a moved block: through `usable_size` and then `free_inline` they + // were resolved twice per moving realloc. The old block stays live until + // that free, so its page cannot change in between. + let seg = segment_of(p); + // SAFETY: `p` is a live block of ours (contract), so `seg` is its live + // segment and `pg` its page; `usable_size`'s own two arms. + let (pg, usable) = unsafe { + let pg = page_of(seg, p); + let flags = (*pg).flags.load(Ordering::Relaxed); + let usable = + if flags & (pflags::HAS_ALIGNED | pflags::SINGLE_BLOCK | pflags::HUGE_SEGMENT) == 0 { + (*pg).block_size + } else { + usable_size_slow(pg, p, flags) + }; + (pg, usable) + }; // NOTE: rewriting this as the one-compare unsigned range check // `newsize.wrapping_sub(usable >> 1) <= usable - (usable >> 1)` measured // FLAT — LLVM already emits that shape from the readable form. @@ -1506,7 +1719,7 @@ unsafe fn realloc_live(p: ptr::NonNull, newsize: usize) -> *mut u8 { // `min` goes too. if newsize > usable { // SAFETY: forwarded contract; a growth copies the whole old block. - return unsafe { realloc_move(p, newsize, usable) }; + return unsafe { realloc_move_at(p, newsize, usable, seg, pg) }; } if newsize >= usable / 2 { // Keep in place. NOTE: this counter costs a TLS heap lookup (a call @@ -1526,41 +1739,33 @@ unsafe fn realloc_live(p: ptr::NonNull, newsize: usize) -> *mut u8 { // frame it saved was smaller than the call it added. // A shrink below half: the copy is the new, smaller size. // SAFETY: forwarded contract. - unsafe { realloc_move(p, newsize, newsize) } + unsafe { realloc_move_at(p, newsize, newsize, seg, pg) } } -/// The moving arm of [`realloc_live`]: allocate `newsize`, copy `copy` bytes, -/// free `p`. Inlined into both of its call sites, so each copy length is a -/// value the site already holds. +/// Allocate, copy `copy` bytes and free `p`, for a caller that has resolved +/// `p`'s segment and page. /// /// # Safety -/// `p` live from this allocator; `copy <= min(usable_size(p), newsize)`. +/// `p` is a live block of ours with at least `copy` usable bytes, +/// `copy <= newsize`, `seg == segment_of(p)` and `pg == page_of(seg, p)`. #[inline(always)] -unsafe fn realloc_move(p: *mut u8, newsize: usize, copy: usize) -> *mut u8 { +unsafe fn realloc_move_at( + p: *mut u8, + newsize: usize, + copy: usize, + seg: *mut Segment, + pg: *mut Page, +) -> *mut u8 { let np = malloc(newsize); if np.is_null() { return ptr::null_mut(); } - // SAFETY: both live and disjoint; prefix preserved then p consumed. + // SAFETY: `p` is live and `copy` of its bytes are ours; `np` is a fresh + // block of at least `newsize >= copy` bytes. `p` is freed once, through + // the page resolved by the caller. unsafe { core::ptr::copy_nonoverlapping(p, np, copy); - // `free_inline`, not the outlined `free`: this function has ALREADY - // masked `p` to its segment for `usable_size`, and inlining the free - // here lets LLVM common-subexpression that away instead of masking - // back to the same header a second time. Splitting `free_inline` to - // pass the segment explicitly was tried first and cost +1 Ir on EVERY - // free (batch_lifo 60.00 -> 61.00); letting the inliner find it costs - // other callers nothing because only this one opts in. - // - // REFUTED (2026-09-24): resolving this free's page, flags and owner - // BEFORE the copy, so the opaque `memcpy` would not force LLVM to - // re-derive them after it (five instructions per move). `free` stayed - // byte-identical, but LLVM re-loaded the page index before the copy - // anyway (the `malloc` above may write memory), and holding four - // values across the call took a sixth callee-saved register plus a - // stack adjustment: opscan `realloc` +8.00 (+4 per move), lua - // +240,221 and python +302,857 allocator Ir. - free_inline(p); + free_body::(p, seg, pg); stat_realloc(false); } np diff --git a/crates/rusty_alloc/src/bins.rs b/crates/rusty_alloc/src/bins.rs index 2ac7faa..04ca400 100644 --- a/crates/rusty_alloc/src/bins.rs +++ b/crates/rusty_alloc/src/bins.rs @@ -18,7 +18,7 @@ pub const PAGES_DIRECT: usize = crate::types::SMALL_WSIZE_MAX + 1; /// Map a size to its bin index (`mi_bin`). Sizes above [`MEDIUM_OBJ_SIZE_MAX`] /// map to [`BIN_HUGE`], the dedicated-segment path. #[inline] -pub fn bin(size: usize) -> usize { +pub const fn bin(size: usize) -> usize { let wsize = wsize_from_size(size); if wsize <= 1 { 1 @@ -46,6 +46,39 @@ pub fn bin(size: usize) -> usize { } } +/// [`bin`] of every small size, by its word count: `SMALL_BIN[w]` is +/// `bin(w * INTPTR_SIZE)` for `w <= SMALL_WSIZE_MAX`. +static SMALL_BIN: [u8; crate::types::SMALL_WSIZE_MAX + 1] = { + let mut t = [0u8; crate::types::SMALL_WSIZE_MAX + 1]; + let mut w = 0; + while w <= crate::types::SMALL_WSIZE_MAX { + let b = bin(w * INTPTR_SIZE); + assert!(b <= u8::MAX as usize); + t[w] = b as u8; + w += 1; + } + t +}; + +/// [`bin`], from a table for small sizes. The generic slow path re-derives +/// the bin of a size the fast path already rejected, and on a small size the +/// computed form (two compares, a `bsr`, shifts) was 10.7 Ir per trip +/// (Endless Sky: 70 k trips); the table is a load. `bin(size)` depends on +/// `size` only through its word count when the size is small, so the table +/// is exact. +/// +/// Takes the word count the caller already computed (`w == +/// wsize_from_size(size)`), so the generic path derives it once. +#[inline] +pub fn bin_by_wsize(size: usize, w: usize) -> usize { + debug_assert_eq!(w, wsize_from_size(size)); + if w <= crate::types::SMALL_WSIZE_MAX { + SMALL_BIN[w] as usize + } else { + bin(size) + } +} + /// Block size (bytes) served by `bin` (`_mi_bin_size` inverse of [`bin`]). /// Meaningless for [`BIN_HUGE`]/[`BIN_FULL`]. #[inline] diff --git a/crates/rusty_alloc/src/heap.rs b/crates/rusty_alloc/src/heap.rs index b0f8d3e..68503f0 100644 --- a/crates/rusty_alloc/src/heap.rs +++ b/crates/rusty_alloc/src/heap.rs @@ -725,8 +725,8 @@ impl Heap { } return (r.0, u8::from(r.1)); } - let bin = bins::bin(size); let w = wsize_from_size(size); + let bin = bins::bin_by_wsize(size, w); // SAFETY: `bin` indexes `self.pages`; the queue's first page, if any, // is a live page of this heap. unsafe { diff --git a/crates/rusty_alloc/src/init.rs b/crates/rusty_alloc/src/init.rs index c9673aa..4579bbf 100644 --- a/crates/rusty_alloc/src/init.rs +++ b/crates/rusty_alloc/src/init.rs @@ -1,1250 +1,1297 @@ -//! Thread + heap lifecycle (mirrors upstream `init.c` + the abandonment side -//! of `arena-abandon.c`). -//! -//! Every thread gets a lazily-created [`HeapBox`] (one OS page from the prim -//! layer — never from the allocator itself, so bootstrap can't recurse). The -//! fast path reaches it through a const-init `thread_local!` `Cell` pointer, -//! which the R1 spike measured at atomic-load parity (0.3 ns) — no dtor is -//! registered on that key (Cell is !Drop), so accessing it never allocates. -//! -//! Thread EXIT runs through the prim TLS destructor (`FlsAlloc` / -//! `pthread_key_create`, built in M1 for exactly this): drain delayed frees, -//! retire what died, transition every surviving page to `NEVER` (spinning out -//! in-flight `FREEING` remotes — the loom-modeled teardown order), zero the -//! segments' thread id, and publish them on the global abandoned list. Any -//! other thread's allocation slow path adopts them later. - -use core::cell::{Cell, UnsafeCell}; -use core::ffi::c_void; -use core::ptr; -use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; - -use crate::heap::Heap; -use crate::page::{ - DelayedList, XFLAG_NEVER, page_collect, page_collect_and_set_flag, page_set_flag, -}; -use crate::segment::{self, Segment}; -use crate::{os, prim}; - -/// A heap plus the cross-thread fields remote threads may touch. Remote -/// threads only ever dereference `delayed` (via page `xheap` addresses) — -/// never the heap itself — which is what makes `&mut` on the inner heap sound. -#[repr(C)] -pub struct HeapBox { - /// Cross-thread delayed-free list (remote-reachable). MUST stay the first - /// field: pages store `&delayed` in `xheap`, and owner routing recovers - /// the box by container-of (delayed at offset 0 ⇒ the addresses coincide). - pub delayed: DelayedList, - /// Next box in the global registry (guarded by the registry lock). - /// AtomicPtr, not usize: reachability (miri's leak check and honesty - /// about what is cached vs leaked) follows pointers, not integers. - pub next_box: AtomicPtr, - /// Owning thread id (heaps allocate only from their creating thread). - pub owner_tid: usize, - /// The subprocess this heap's segments are abandoned into. - /// - /// Mirrored from the `SUBPROC` thread-local so that TEARDOWN never has to - /// read it. `thread_done` runs from a platform TLS destructor - /// (`pthread_key_create` / `FlsAlloc`), and the destruction order between - /// *that* callback and Rust's own `thread_local!` storage is unspecified on - /// every platform. On aarch64-apple-darwin it is observably wrong: - /// `my_subproc()` reads back 0 inside `thread_done`, so a tagged thread's - /// segments landed in the MAIN subprocess and `mi_subproc_*` isolation was - /// silently lost — the abandon fired correctly, only its destination was - /// wrong. The heap box is our own mapping and is alive for the whole of - /// teardown (it is the value passed *to* the destructor), so reading the tag - /// from here is correct on every target instead of accidentally correct on - /// one. - pub subproc: AtomicUsize, - /// Heap tag (`mi_heap_new_ex`; reported by the block visitor). - pub tag: i32, - /// Whether `mi_heap_destroy` may drop live blocks (else it delegates to - /// delete semantics). - pub allow_destroy: bool, - /// The owner-only heap state. - pub heap: UnsafeCell, -} - -const _: () = assert!(core::mem::offset_of!(HeapBox, delayed) == 0); - -/// Wrapper so the empty sentinel box can be a `static`. -#[repr(transparent)] -pub struct EmptyHeapBox(HeapBox); - -// SAFETY: the sentinel is never written and never escapes to remote threads: -// no page ever stores its `delayed` address in `xheap` (pages are re-homed -// only by REAL heaps), its `owner_tid` of 0 matches no live thread, and every -// reader goes through raw-pointer READS — `heap_box_fast` documents that no -// `&mut` may ever be formed to it. It exists purely so the malloc fast path -// has a valid heap to read before this thread has created one. -unsafe impl Sync for EmptyHeapBox {} - -/// The "no heap yet" sentinel: every `direct` slot holds the empty PAGE -/// sentinel, so a fresh thread's first malloc falls through to the generic -/// path — which detects this box and performs the real thread init — without -/// the fast path ever testing for it. This is upstream's `_mi_heap_empty` -/// trick, and the exact design our own [`crate::page::EMPTY_PAGE`] already -/// uses one level down: replace a null-test on the hot path with a sentinel -/// whose data routes the miss to the slow path for free. -/// -/// `export_name` (not mangled) because the x86-64 Linux TLS slot's `.tdata` -/// initializer below names it in assembly; `.hidden` there keeps it out of -/// the cdylib's export table. -/// -/// **In flash on a one-region target.** The sentinel is read and never -/// written — `heap_box_fast` documents that contract, the fast path only -/// writes after a non-null pop and every `direct` slot here points at the -/// empty page whose free list is null, and no page ever carries its address -/// in `xheap`. Its type has interior mutability (a `DelayedList`, an -/// `AtomicPtr`, the `UnsafeCell`), which is why rustc places it in -/// `.data` by default: 1,752 bytes paid once in flash for the image and once -/// more in RAM for the loader's copy, the largest single item left on the -/// ESP32-S3 after 2.0.2 and RAM the linker takes from the stack. Placing it -/// in `.rodata.*` keeps the flash copy and drops the RAM one. Any write -/// would fault loudly against the flash cache, which is the right way for -/// the contract to fail. Bare metal only — a hosted target may need to -/// relocate the pointers inside it at load, and has RAM to spare. -#[cfg_attr( - all( - ra_single_threaded, - not(miri), - not(windows), - not(unix), - not(target_arch = "wasm32") - ), - unsafe(link_section = ".rodata.rusty_alloc_empty_heap_box") -)] -#[unsafe(export_name = "__ra_empty_heap_box")] -pub static EMPTY_HEAP_BOX: EmptyHeapBox = EmptyHeapBox(HeapBox { - delayed: DelayedList::new(), - next_box: AtomicPtr::new(ptr::null_mut()), - owner_tid: 0, - subproc: AtomicUsize::new(0), - tag: 0, - allow_destroy: false, - heap: UnsafeCell::new(Heap::new()), -}); - -/// Pointer to the shared empty heap box. -#[inline] -pub const fn empty_heap_box_ptr() -> *mut HeapBox { - (&raw const EMPTY_HEAP_BOX.0).cast_mut() -} - -/// Recover the owning HeapBox from a page's `xheap` value (the container-of -/// trick over the offset-0 delayed list). -/// -/// # Safety -/// `xheap` must be a live HeapBox's delayed-list address (nonzero). -#[inline] -pub unsafe fn box_of_xheap(xheap: usize) -> *mut HeapBox { - debug_assert!(xheap != 0); - xheap as *mut HeapBox -} - -// Global registry of live heap boxes: keeps every heap (and through it every -// segment) reachable-from-static — the process-wide stats walk of M7, and -// what makes miri's reachability-based leak check reflect reality (an -// allocator's arenas at process exit are cached state, not leaks). -static HEAPS_LOCK: AtomicBool = AtomicBool::new(false); -static HEAPS_HEAD: AtomicPtr = AtomicPtr::new(ptr::null_mut()); - -fn heaps_lock() { - while HEAPS_LOCK - .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) - .is_err() - { - core::hint::spin_loop(); - } -} - -fn heaps_unlock() { - HEAPS_LOCK.store(false, Ordering::Release); -} - -fn heaps_register(hb: *mut HeapBox) { - heaps_lock(); - // SAFETY: hb is fully initialized and not yet published. - unsafe { - (*hb) - .next_box - .store(HEAPS_HEAD.load(Ordering::Relaxed), Ordering::Relaxed); - } - HEAPS_HEAD.store(hb, Ordering::Relaxed); - heaps_unlock(); -} - -fn heaps_unregister(hb: *mut HeapBox) { - heaps_lock(); - let mut cur = HEAPS_HEAD.load(Ordering::Relaxed); - if cur == hb { - // SAFETY: links guarded by the lock. - unsafe { - HEAPS_HEAD.store((*hb).next_box.load(Ordering::Relaxed), Ordering::Relaxed); - } - } else { - while !cur.is_null() { - // SAFETY: links guarded by the lock; nodes are live boxes. - unsafe { - let nxt = (*cur).next_box.load(Ordering::Relaxed); - if nxt == hb { - (*cur) - .next_box - .store((*nxt).next_box.load(Ordering::Relaxed), Ordering::Relaxed); - break; - } - cur = nxt; - } - } - } - heaps_unlock(); -} - -/// Unlink and return one live heap box owned by `tid`, or null. -/// -/// The lock is released before return so the caller can abandon the heap -/// without re-entering `HEAPS_LOCK` (abandon walks pages and can allocate). -fn take_heap_owned_by(tid: usize) -> *mut HeapBox { - heaps_lock(); - let mut prev: *mut HeapBox = ptr::null_mut(); - let mut cur = HEAPS_HEAD.load(Ordering::Relaxed); - while !cur.is_null() { - // SAFETY: links and boxes are live under the lock. - unsafe { - let nxt = (*cur).next_box.load(Ordering::Relaxed); - if (*cur).owner_tid == tid { - if prev.is_null() { - HEAPS_HEAD.store(nxt, Ordering::Relaxed); - } else { - (*prev).next_box.store(nxt, Ordering::Relaxed); - } - heaps_unlock(); - return cur; - } - prev = cur; - cur = nxt; - } - } - heaps_unlock(); - ptr::null_mut() -} - -/// Storage for the calling thread's default-heap pointer. -/// -/// On x86-64 Linux this is a RAW ELF TLS symbol reached with the -/// **initial-exec** model rather than `thread_local!`'s general-dynamic. That -/// was the last `__tls_get_addr` call on the malloc fast path — 7.3 M -/// instructions on perl, a third of everything still separating us from -/// mimalloc after the M10b free-path bricks. Stable Rust has no -/// `-Z tls-model`, so the symbol is declared in `global_asm!` and the linker -/// resolves its offset. -/// -/// **Why this is sound where a thread-pointer-keyed side table is not.** The -/// storage IS the thread's own TLS block. Every thread gets a fresh block -/// initialised from the `.tdata` template (the empty-heap sentinel) when it -/// is created, so a recycled TCB cannot hand a new thread a dead thread's -/// heap — the staleness -/// question that made the keyed-cache design a P0 risk simply does not arise -/// here. What initial-exec costs instead is a LOAD-TIME constraint: it needs a -/// slot in the static TLS block, so a `dlopen` long after startup could fail -/// to load us. That fails loudly at load time rather than corrupting memory, -/// and it is the same trade upstream mimalloc ships -/// (`__attribute__((tls_model("initial-exec")))` on `_mi_heap_default`). -#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] -mod heap_tls { - use super::HeapBox; - - // An 8-byte thread-local slot. `.tdata` (not `.tbss`): the slot is - // INITIALISED to the empty-heap sentinel, so every new thread reads a - // valid heap pointer from its very first malloc and the fast path needs - // no "not yet created" test at all — the sentinel's empty direct table - // routes the first allocation to the generic path, which performs the - // real init there (upstream's `_mi_heap_empty` design). The loader - // relocates the `.quad` in the TLS template once at load time; every - // thread's block is then copied from that template. `.hidden` makes both - // symbols non-preemptible, which is what permits the initial-exec - // (GOTTPOFF) relocation instead of a general-dynamic call, and keeps the - // sentinel out of the cdylib's export table. - core::arch::global_asm!( - ".section .tdata,\"awT\",@progbits", - ".globl __ra_tls_heap", - ".hidden __ra_tls_heap", - ".hidden __ra_empty_heap_box", - ".type __ra_tls_heap,@object", - ".p2align 3", - "__ra_tls_heap:", - ".quad __ra_empty_heap_box", - ".size __ra_tls_heap,8", - ".text", - ); - - /// Address of THIS thread's slot: the thread pointer plus the offset the - /// dynamic linker wrote into the GOT. - #[inline(always)] - fn slot() -> *mut *mut HeapBox { - let off: usize; - // SAFETY: reads the GOT entry the dynamic linker relocated at load - // time, before any of our code can run, and never writes. It is - // constant for the life of the process, hence `pure` + `readonly` — - // that is what lets LLVM hoist and CSE the address computation - // instead of treating the block as an optimisation barrier. - unsafe { - core::arch::asm!( - "mov {o}, qword ptr [rip + __ra_tls_heap@GOTTPOFF]", - o = out(reg) off, - options(nostack, preserves_flags, readonly, pure), - ); - } - // `thread_id()` IS the fs base on this target. TLS offsets are - // negative there (the block grows down from the thread pointer), so - // this adds a two's-complement offset and must wrap. - core::ptr::with_exposed_provenance_mut(super::thread_id().wrapping_add(off)) - } - - /// Read this thread's heap pointer in TWO instructions. - /// - /// The obvious form — `slot().read()` — costs FOUR: load the offset from - /// the GOT, read the thread pointer from `fs:0`, add them, dereference. - /// But x86 performs that addition IN THE ADDRESSING MODE: `fs:[reg]` is a - /// segment-relative load, so the explicit `fs:0` read and the add both - /// disappear. Measured at 4.00 Ir per malloc before this change — a - /// quarter of our entire malloc-side deficit spent just locating the heap. - #[inline(always)] - pub fn get() -> *mut HeapBox { - let v: *mut HeapBox; - // SAFETY: reads the GOT entry the loader relocated at startup, then - // loads 8 aligned bytes of this thread's own TLS block through the fs - // segment. Both are reads; nothing is written. - unsafe { - core::arch::asm!( - "mov {t}, qword ptr [rip + __ra_tls_heap@GOTTPOFF]", - "mov {o}, qword ptr fs:[{t}]", - t = out(reg) _, - o = out(reg) v, - options(nostack, preserves_flags, readonly), - ); - } - v - } - - #[inline(always)] - pub fn set(hb: *mut HeapBox) { - // SAFETY: as `get` — the slot is owned exclusively by this thread. - unsafe { slot().write(hb) } - } -} - -/// Every other target keeps the `thread_local!`: Windows TLS is an index into -/// a per-thread array (no `__tls_get_addr` to remove), and miri models the -/// macro but not a hand-rolled TLS symbol. -#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))] -mod heap_tls { - use super::HeapBox; - use core::cell::Cell; - - ra_thread_local! { - /// Fast-path heap pointer. Const-init + !Drop ⇒ plain TLS access, no - /// lazy-init branch, no allocation ever. Initialised to the - /// empty-heap SENTINEL (never null) so the malloc fast path can read - /// through it unconditionally; see `EMPTY_HEAP_BOX`. - static HEAP_PTR: Cell<*mut HeapBox> = const { Cell::new(super::empty_heap_box_ptr()) }; - } - - #[inline(always)] - pub fn get() -> *mut HeapBox { - HEAP_PTR.with(|c| c.get()) - } - - #[inline(always)] - pub fn set(hb: *mut HeapBox) { - HEAP_PTR.with(|c| c.set(hb)); - } -} - -ra_thread_local! { - /// Cached OS thread id. `free` needs the calling thread's id on EVERY - /// call to route local-vs-remote; the raw `prim::thread_id()` is a libc - /// call (`pthread_self` through the PLT from a cdylib) measured at - /// 1.41 ns vs 0.25 ns for this cache — ~18-20% of a whole malloc+free - /// pair (M9 probe, both platforms). 0 = not yet cached; no real thread id - /// is 0 on either platform. - static TID: Cell = const { Cell::new(0) }; -} - -/// The calling thread's id — read STRAIGHT FROM THE THREAD-POINTER REGISTER -/// where the target allows it. -/// -/// Why this and not `thread_local!`: the shipping artifact is a **cdylib** -/// (LD_PRELOAD). Rust's `thread_local!` in a shared library compiles to the -/// general-dynamic TLS model, so every access is a CALL into `ld.so`'s -/// `__tls_get_addr` — the per-function instruction profile put that call at -/// 12.97 M Ir, 1.96% of a whole lua run, comparable to our entire `free`. -/// (The earlier ns-level probe missed it because it measured TLS inside an -/// EXECUTABLE, where the model is local-exec: a register offset, no call. -/// Measure the artifact you ship.) -/// -/// A ceiling probe with `-Z tls-model=initial-exec` valued removing those -/// calls at 2.00% of our instructions and 33% of the gap to mimalloc; this -/// gets the same effect on stable, for the id, with no linker involvement — -/// the thread pointer is one register read. It is exactly what mimalloc's -/// `_mi_thread_id()` does. -/// -/// Uniqueness: the value is the thread-control-block address, unique among -/// LIVE threads. It can be recycled after a thread dies, which is sound here -/// for the same reason it is in mimalloc: a dying thread abandons its -/// segments (`thread_id` stored as 0) before its TCB can be reused, so a -/// recycled id can never match a stale segment. -#[inline(always)] -pub fn thread_id() -> usize { - #[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] - { - let tp: usize; - // SAFETY: reads the TCB self-pointer at fs:0, which the x86-64 Linux - // ABI guarantees is present and thread-unique. No memory is written. - unsafe { - core::arch::asm!("mov {}, fs:0", out(reg) tp, - options(nostack, preserves_flags, readonly)); - } - tp - } - #[cfg(all(target_arch = "x86_64", target_os = "windows", not(miri)))] - { - let teb: usize; - // SAFETY: gs:0x30 is the TEB self-pointer on x86-64 Windows — - // present in every thread, unique, read-only here. - unsafe { - core::arch::asm!("mov {}, gs:0x30", out(reg) teb, - options(nostack, preserves_flags, readonly)); - } - teb - } - // AArch64 splits by OS: the register holding the thread pointer is NOT the - // same one on Darwin as everywhere else. - #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple"), not(miri)))] - { - let tp: usize; - // SAFETY: on the standard AArch64 ABI (Linux, Android, BSD) tpidr_el0 - // IS the thread pointer; read-only. - unsafe { - core::arch::asm!("mrs {}, tpidr_el0", out(reg) tp, - options(nostack, preserves_flags, readonly)); - } - tp - } - #[cfg(all(target_arch = "aarch64", target_vendor = "apple", not(miri)))] - { - let tp: usize; - // SAFETY: read-only system-register read. - // - // DARWIN IS NOT THE STANDARD AArch64 ABI. Apple puts the thread pointer - // in tpidrRO_el0 and uses tpidr_el0 for the CPU/cluster id. Reading - // tpidr_el0 here — as every other AArch64 target correctly does — was a - // memory-safety bug, measured on macOS 26 / M-series: - // - // * tpidr_el0 returns small values (0x1002, 0x2005, …), not pointers; - // * it CHANGES under a single thread as that thread migrates between - // cores (5 distinct values over 3M reads on the main thread); - // * and 8 live threads produced only 5 distinct values — DIFFERENT - // THREADS COLLIDED. - // - // thread_id() is the allocator's ownership identity: `segment.thread_id` - // decides whether a free takes the local (unsynchronised) path or the - // remote one. Colliding ids let one thread free into another thread's - // segment through the owner path, racing non-atomic page state — heap - // corruption, not merely a lost optimisation. A drifting id separately - // makes a thread stop recognising its own segments. - // - // tpidrro_el0 is the real thread pointer: stable within a thread and - // unique across live threads (it is `pthread_self() + 0xe0`). Apple - // documents the LOW 3 BITS as the current CPU number, so they must be - // masked off — that is precisely the drift observed above. - unsafe { - core::arch::asm!("mrs {}, tpidrro_el0", out(reg) tp, - options(nostack, preserves_flags, readonly)); - } - tp & !0b111 - } - // Every other target: the cached-TLS path (still far cheaper than asking - // the OS on each call). - #[cfg(not(any( - all(target_arch = "x86_64", target_os = "linux", not(miri)), - all(target_arch = "x86_64", target_os = "windows", not(miri)), - all(target_arch = "aarch64", not(miri)) - )))] - { - let t = TID.with(|c| c.get()); - if t != 0 { t } else { init_tid() } - } -} - -#[cold] -#[cfg_attr( - any( - all(target_arch = "x86_64", target_os = "linux", not(miri)), - all(target_arch = "x86_64", target_os = "windows", not(miri)), - all(target_arch = "aarch64", not(miri)) - ), - allow(dead_code) -)] -fn init_tid() -> usize { - let t = prim::thread_id(); - TID.with(|c| c.set(t)); - t -} - -/// Snapshot-visit every registered heap (stats aggregation). The visitor -/// receives a COPY of the heap's counters. -/// -/// **This read is racy by design and `volatile` does not make it less so** — -/// clippy's `volatile_composites` is right to flag it, and the lint is -/// allowed HERE rather than workspace-wide so the reasoning stays attached to -/// the one site that does it. `read_volatile` on a composite gives no -/// per-field atomicity and no ordering, so a concurrent owner mutating its -/// counters can yield a snapshot whose fields disagree with each other. That -/// is accepted: these are `#[cfg(debug_assertions)]` diagnostic counters -/// (upstream's `MI_STAT` rule), nothing branches on them, and the alternative -/// — taking `&` to a struct another thread is writing — would be UB rather -/// than merely imprecise. The real fix is making the counters atomics, which -/// is the tracked follow-up (R-005 in the hardening audit). -#[allow( - clippy::volatile_composites, - reason = "racy diagnostic snapshot; see the doc comment. Tracked as R-005." -)] -pub fn for_each_heap(f: &mut dyn FnMut(&crate::heap::Heap)) { - heaps_lock(); - let mut hb = HEAPS_HEAD.load(Ordering::Acquire); - while !hb.is_null() { - // SAFETY: registry entries are live boxes. The read is unsynchronised - // with the owning thread's writes (see above) but stays within a live, - // correctly-aligned allocation for the whole of the copy. - unsafe { - let snapshot = core::ptr::read_volatile((*hb).heap.get()); - f(&snapshot); - hb = (*hb).next_box.load(Ordering::Acquire); - } - } - heaps_unlock(); -} - -/// Get (or create) the calling thread's heap box. NULL when creation fails -/// (memory exhaustion) — callers translate that into their own failure value -/// (a null allocation, a no-op collect); the TLS slot keeps the sentinel so a -/// later call retries. -#[inline] -pub fn heap_box() -> *mut HeapBox { - let hb = heap_tls::get(); - if hb == empty_heap_box_ptr() { - init_thread_heap() - } else { - hb - } -} - -/// This thread's heap box WITHOUT the created-yet test — may return the -/// shared empty sentinel. For the malloc fast path only. -/// -/// # Safety (for callers) -/// The returned box may be [`EMPTY_HEAP_BOX`], which is shared between every -/// uninitialised thread: callers must perform RAW-POINTER READS ONLY and must -/// never form a `&mut` (or write) through it. Route any miss to -/// [`ensure_heap`] before mutating anything. -#[inline] -pub fn heap_box_fast() -> *mut HeapBox { - heap_tls::get() -} - -/// Upgrade a possibly-sentinel box from [`heap_box_fast`] to this thread's -/// real, initialised heap box. NULL when creation fails (see [`heap_box`]). -#[inline] -pub fn ensure_heap(hb: *mut HeapBox) -> *mut HeapBox { - if hb == empty_heap_box_ptr() { - init_thread_heap() - } else { - hb - } -} - -/// Allocate + initialize a HeapBox (shared by thread bootstrap and -/// `mi_heap_new*`). Registered in the global registry; NOT installed in TLS. -/// -/// Returns NULL when the OS refuses the mapping (memory exhaustion). This -/// used to `expect()` — which, under the release profile's `panic = "abort"`, -/// turned OOM into a process abort on a path reachable from any fresh -/// thread's first `malloc`. The malloc contract on OOM is a null return, and -/// upstream returns NULL from `mi_heap_new` for the same reason (H-18/R-003 -/// in the hardening audit). -pub fn create_heap(tag: i32, allow_destroy: bool, arena_id: i32) -> *mut HeapBox { - // First-class heaps are not stored in `done_slot`. A thread that only - // ever called `mi_heap_new` / `heap_malloc` never ran `init_thread_heap`, - // so the TLS destructor never fired and leftover heaps stayed DELAYED - // (OH-rusty_alloc-13). Bootstrap the backing heap (and the exit hook) - // first. `init_thread_heap` calls `create_heap_uninstalled`, not this - // function, so there is no recursion. - if !crate::ONE_THREAD && heap_tls::get() == empty_heap_box_ptr() { - let _ = heap_box(); - } - create_heap_uninstalled(tag, allow_destroy, arena_id) -} - -fn create_heap_uninstalled(tag: i32, allow_destroy: bool, arena_id: i32) -> *mut HeapBox { - let size = core::mem::size_of::(); - // On a one-region target the FIRST descriptor is a static of the fixed - // backend's, so that a firmware's region is whole segments and needs no - // page carved out of it — the page is what made an aligned container of - // the region pad itself out to the next segment - // (`docs/plans/finished/region-alignment-bug.md` §7). Every later heap, - // and every heap on a hosted target, takes a page as before. - let first = if crate::ONE_REGION { - prim::fixed::take_first_heap_box() - } else { - None - }; - let hb: *mut HeapBox = match first { - Some(p) => p, - None => { - let Ok(block) = os::alloc_aligned(size, os::page_size(), true, false) else { - return ptr::null_mut(); - }; - block.ptr.cast() - } - }; - // SAFETY: fresh committed mapping large enough for HeapBox; we initialize - // every field before publishing the pointer. - unsafe { - ptr::write( - hb, - HeapBox { - delayed: DelayedList::new(), - next_box: AtomicPtr::new(ptr::null_mut()), - owner_tid: thread_id(), - // Captured HERE, on the live creating thread, where the - // thread-local is unambiguously valid. - subproc: AtomicUsize::new(my_subproc()), - tag, - allow_destroy, - // A new heap starts as a copy of the empty one. On a - // one-region target that copy is taken from the sentinel - // itself, which already holds the whole template in flash; - // materialising `Heap::new()` here as well cost a second - // ~900-byte `.rodata` blob for the bin table. Hosted builds - // keep the constant, whose codegen was measured in Ir. - heap: UnsafeCell::new(if crate::ONE_REGION { - // SAFETY (within the enclosing block): a raw, - // non-overlapping read of the never-written sentinel's - // heap — the only access `heap_box_fast`'s contract - // permits, and `Heap` owns nothing that a bitwise copy - // could double-free. - ptr::read((*empty_heap_box_ptr()).heap.get()) - } else { - Heap::new() - }), - }, - ); - (*(*hb).heap.get()).delayed = &raw const (*hb).delayed; - (*(*hb).heap.get()).arena_id = arena_id; - (*(*hb).heap.get()).tag = tag; - // Per-heap CSPRNG stream (free-list keys, guarded sampling) -- seeded - // only where something draws from it. A build with neither `secure` - // nor guard pages has no consumer, and seeding still cost the ChaCha - // block function plus the splitmix fallback in flash. - if crate::RNG_USED { - (*(*hb).heap.get()).rng.reseed(); - } - // Guarded objects follow the option table (MI_GUARDED-equivalent) -- - // where a guard page can exist at all. `GUARD_PAGES` first, so that on - // a target without one the option reads fold away with the sampler - // (`docs/plans/finished/firmware-code-size.md`, lever 3). - let rate = crate::options::get(33).max(0) as usize; // guarded_sample_rate - let gmin = crate::options::get(30).max(0) as usize; // guarded_min - let gmax = crate::options::get(31).max(0) as usize; // guarded_max - if crate::GUARD_PAGES && gmax > 0 { - (*(*hb).heap.get()).guarded_set_size_bound(gmin, gmax); - (*(*hb).heap.get()) - .guarded_set_sample_rate(rate, crate::options::get(34).max(0) as usize); - } - } - heaps_register(hb); - hb -} - -#[cold] -fn init_thread_heap() -> *mut HeapBox { - let hb = create_heap_uninstalled(0, false, -1); - if hb.is_null() { - // OOM: install nothing — the TLS slot keeps the sentinel, so the - // NEXT allocation retries creation instead of dereferencing null. - return ptr::null_mut(); - } - heap_tls::set(hb); - BACKING_PTR.with(|c| c.set(hb)); - // The exit hook exists to abandon this heap when its thread ENDS. On a - // single-context build no thread ends, the backend's TLS never runs a - // destructor anyway (`prim::fixed::tls_new` discards it), and creating - // the slot was a compare-and-swap, a spin and a table entry paid once for - // nothing (`docs/plans/finished/firmware-code-size.md`, lever 2). - if !crate::ONE_THREAD { - done_slot().set(hb.cast::()); - } - hb -} - -ra_thread_local! { - /// The thread's original (backing) heap — what `mi_heap_get_backing` - /// returns regardless of `mi_heap_set_default` swaps. - static BACKING_PTR: Cell<*mut HeapBox> = const { Cell::new(ptr::null_mut()) }; -} - -/// `mi_heap_get_backing`. -pub fn backing_heap() -> *mut HeapBox { - let b = BACKING_PTR.with(|c| c.get()); - if b.is_null() { - let _ = heap_box(); // bootstraps and sets BACKING_PTR - BACKING_PTR.with(|c| c.get()) - } else { - b - } -} - -/// `mi_heap_set_default`: install `hb` as this thread's default; returns the -/// previous default. -/// -/// # Safety -/// `hb` must be a live heap owned by the calling thread. -pub unsafe fn set_default_heap(hb: *mut HeapBox) -> *mut HeapBox { - let prev = heap_box(); - heap_tls::set(hb); - prev -} - -/// The process-wide TLS slot whose destructor abandons a dying thread's heap. -fn done_slot() -> prim::TlsSlot { - static RAW: AtomicUsize = AtomicUsize::new(0); - static INIT: AtomicBool = AtomicBool::new(false); - let raw = RAW.load(Ordering::Acquire); - if raw != 0 { - // SAFETY: RAW holds an into_raw of a slot created below. - return unsafe { prim::TlsSlot::from_raw(raw - 1) }; - } - if INIT - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - { - // A panic here would be wrong twice over. An allocator must not unwind - // into its C callers (which is why the release profile is - // panic=abort), AND the winner of this race is the only thread that - // ever publishes RAW — so unwinding past the store below would strand - // every other thread in the spin loop FOREVER, turning a rare - // resource failure into a silent process-wide hang. Fail loudly and - // deterministically instead, identically in debug and release. - let Some(slot) = prim::TlsSlot::new(Some(thread_done_cb)) else { - crate::abort(); - }; - RAW.store(slot.into_raw() + 1, Ordering::Release); // +1: 0 is the unset sentinel - } - loop { - let raw = RAW.load(Ordering::Acquire); - if raw != 0 { - // SAFETY: as above. - return unsafe { prim::TlsSlot::from_raw(raw - 1) }; - } - core::hint::spin_loop(); - } -} - -#[cfg(all(windows, not(miri)))] -unsafe extern "system" fn thread_done_cb(v: *const c_void) { - if !v.is_null() { - // SAFETY: v is the HeapBox this thread stored in the done slot. - unsafe { thread_done(v.cast_mut().cast()) }; - } -} - -#[cfg(any(not(windows), miri))] -unsafe extern "C" fn thread_done_cb(v: *mut c_void) { - if !v.is_null() { - // SAFETY: v is the HeapBox this thread stored in the done slot. - unsafe { thread_done(v.cast()) }; - } -} - -/// Abandon the dying thread's heap. Teardown order is the loom-modeled one: -/// (1) every surviving page → NEVER (spins out FREEING remotes), (2) THEN -/// drain the delayed list, (3) THEN release the heap storage. -/// -/// # Safety -/// Must run on the dying thread with `hb` its live heap box, exactly once. -pub unsafe fn thread_done(hb: *mut HeapBox) { - // SAFETY: per contract `hb` is this thread's live box; `owner_tid` is a - // plain field. - let tid = unsafe { (*hb).owner_tid }; - // SAFETY: per contract we are the owner and this runs exactly once; - // sequencing per the model. - unsafe { thread_done_one(hb) }; - // Back to the SENTINEL, not null — the slot is never null, so a - // post-teardown malloc re-enters the generic path and re-initialises. - heap_tls::set(empty_heap_box_ptr()); - // First-class heaps this thread created are not stored in `done_slot` - // (`set_default_heap` does not update it). Leaving them DELAYED meant a - // second remote free after thread exit returned (OH-rusty_alloc-13). - // Abandon every remaining heap we still own so pages go NEVER. Iterative: - // a dying thread can own many first-class heaps; do not recurse. - if !crate::ONE_THREAD { - // Stall guard: a dying thread that keeps re-initialising (empty TLS - // malloc) must not loop forever. Same order as `collect_inner`'s - // `MAX_RECLAIM`. - const MAX_OWNED_HEAPS: usize = 1 << 20; - for _ in 0..MAX_OWNED_HEAPS { - let extra = take_heap_owned_by(tid); - if extra.is_null() { - break; - } - // SAFETY: extra is a live box this dying thread still owns. - unsafe { thread_done_one(extra) }; - } - } -} - -/// Abandon one heap box (NEVER + delayed drain + unregister + free the box). -/// -/// # Safety -/// `hb` is a live heap box owned by the calling thread, not used again. -unsafe fn thread_done_one(hb: *mut HeapBox) { - // SAFETY: per contract we are the owner; sequencing per the model. - unsafe { - // Read the subprocess tag from the BOX, not from `my_subproc()`. We are - // inside a platform TLS destructor and Rust's `thread_local!` storage - // may already be torn down — on aarch64-apple-darwin it reads back 0, - // which silently dropped every tagged thread's segments into the main - // subprocess. Clamped because a corrupt tag must not index out of - // bounds on the teardown path. - let sp = (*hb).subproc.load(Ordering::Acquire).min(MAX_SUBPROCS - 1); - let h = &mut *(*hb).heap.get(); - // Retire everything that is already dead (also drains delayed once). - // TEARDOWN variant: must NOT adopt orphans into a heap we are about to - // destroy — that was the 0.3.1 segfault. - h.collect_for_teardown(); - // Walk remaining segments: transition pages to NEVER, disown, publish. - let mut seg = h.segments; - h.segments = ptr::null_mut(); - while !seg.is_null() { - let next = (*seg).next; - let end = (*seg).next_free_slice as usize; - let mut idx = segment::HEADER_SLICES; - // Running pointer, not `pages[idx]`: `Page` is not a power-of-two - // size, so indexing costs a multiply plus a bounds check on every - // carved slice of every segment a dying thread owns. - let base: *mut crate::page::Page = (&raw mut (*seg).pages).cast(); - let mut slot: *mut crate::page::Page = base.add(idx); - while idx < end { - let len = ((*slot).slice_count as usize).max(1); - if (*slot).block_size > 0 { - // ONE locked read-modify-write, not two: taking the - // cross-thread chain and publishing NEVER are the same - // word, and doing them separately paid two CAS loops per - // live page of every segment a dying thread owns. - page_collect_and_set_flag(slot, XFLAG_NEVER); - (*slot).xheap.store(0, Ordering::Release); - } - idx += len; - // `wrapping_add`, not `add`: the pointer is only ever - // DEREFERENCED while `idx < end` holds, which the span-partition - // invariant makes in-bounds — but the final advance of the loop - // computes a pointer that may be one past the carved region, and - // forming that with `add` would be UB even unread. - slot = slot.wrapping_add(len); - } - if (*seg).used_pages == 0 { - let _ = segment::segment_free(seg); - h.stats.segments_freed += 1; - } else { - (*seg).thread_id.store(0, Ordering::Release); - abandoned_push(seg, sp); - } - seg = next; - } - // Huge segments: same teardown shape (collect NEVER-bound frees, - // release dead ones, abandon the rest). - let mut hseg = h.huge_segments; - h.huge_segments = ptr::null_mut(); - while !hseg.is_null() { - let next = (*hseg).next; - let pg = &raw mut (*hseg).pages[1]; - page_collect(pg); - if (*pg).used == 0 { - let _ = segment::huge_free(hseg); - h.stats.segments_freed += 1; - } else { - page_set_flag(pg, XFLAG_NEVER); - (*pg).xheap.store(0, Ordering::Release); - (*hseg).thread_id.store(0, Ordering::Release); - abandoned_push(hseg, sp); - } - hseg = next; - } - // Post-NEVER drain: blocks a FREEING remote landed while we walked. - // Their pages are abandoned now, so free_local would be wrong — but - // these blocks belong to pages we JUST abandoned; push them back on - // their pages' own lists (flag NEVER routes them there). - let mut b = (*hb).delayed.head.swap(0, Ordering::AcqRel) as *mut crate::page::Block; - while !b.is_null() { - let next = (*b).next; - let seg = segment::segment_of(b.cast()); - let pg = segment::page_of(seg, b.cast()); - crate::page::remote_free(pg, b); - b = next; - } - // Release the heap storage and clear the fast-path pointer. The - // first descriptor on a one-region target is a static, not a page: - // nothing to return (and this path is unreachable there anyway). - heaps_unregister(hb); - if !(crate::ONE_REGION && prim::fixed::is_first_heap_box(hb)) { - let size = core::mem::size_of::(); - let blockdesc = os::OsBlock { - ptr: hb.cast(), - size: os::page_align_up(size), - is_large: false, - is_zero: false, - }; - let _ = os::free(blockdesc); - } - } -} - -/// `mi_heap_delete`: free the heap structure, MIGRATING its live pages and -/// segments into the thread's backing heap (blocks stay valid). -/// -/// # Safety -/// `hb` must be a live first-class heap owned by the calling thread, not the -/// backing heap, used never again. -pub unsafe fn heap_delete(hb: *mut HeapBox) { - let backing = backing_heap(); - if hb == backing { - return; // deleting the backing heap is a no-op (upstream-compatible) - } - // SAFETY: owner thread per contract; absorb everything into backing. - unsafe { - debug_assert_eq!((*hb).owner_tid, thread_id()); - let h = &mut *(*hb).heap.get(); - let bh = &mut *(*backing).heap.get(); - // Teardown: this heap is being deleted, so it must not adopt orphans. - h.collect_for_teardown(); - let mut seg = h.segments; - h.segments = ptr::null_mut(); - while !seg.is_null() { - let next = (*seg).next; - // `next` is read BEFORE adopting and `seg` is not touched after, so - // a release during adoption is safe here. - let _ = bh.adopt_segment(seg); // re-homes queues + xheap (tid ours) - seg = next; - } - let mut hseg = h.huge_segments; - h.huge_segments = ptr::null_mut(); - while !hseg.is_null() { - let next = (*hseg).next; - let pg = &raw mut (*hseg).pages[1]; - (*pg).xheap.store(bh.delayed as usize, Ordering::Release); - (*hseg).next = bh.huge_segments; - bh.huge_segments = hseg; - hseg = next; - } - // Fold the dying heap's counters into backing so process totals hold. - merge_stats(&mut bh.stats, &h.stats); - release_heap_box(hb); - } -} - -/// `mi_heap_destroy`: drop EVERY block of the heap at once and release its -/// memory. Falls back to delete semantics when the heap was created with -/// `allow_destroy = false`. -/// -/// # Safety -/// As [`heap_delete`], and additionally no block of this heap may be used -/// afterwards (the C contract). -pub unsafe fn heap_destroy(hb: *mut HeapBox) { - // SAFETY: owner thread per contract. - unsafe { - if !(*hb).allow_destroy { - heap_delete(hb); - return; - } - debug_assert_eq!((*hb).owner_tid, thread_id()); - let h = &mut *(*hb).heap.get(); - // Spin out any in-flight FREEING remotes before the box dies; blocks - // they queued are dying with the heap, so the delayed list is simply - // discarded. - let mut seg = h.segments; - while !seg.is_null() { - let end = (*seg).next_free_slice as usize; - let mut idx = segment::HEADER_SLICES; - // Same running-pointer walk as `thread_done`'s — the sibling site, - // checked because a defect in one slice walk is a defect in every - // slice walk. Not exercised by `mleak` (this is `heap_destroy`, not - // thread teardown), so it carries no measurement of its own; it is - // the identical transformation on identical code. - let base: *mut crate::page::Page = (&raw mut (*seg).pages).cast(); - let mut slot: *mut crate::page::Page = base.add(idx); - while idx < end { - let len = ((*slot).slice_count as usize).max(1); - if (*slot).block_size > 0 { - page_set_flag(slot, XFLAG_NEVER); - (*slot).xheap.store(0, Ordering::Release); - } - idx += len; - slot = slot.wrapping_add(len); - } - let next = (*seg).next; - let _ = segment::segment_free(seg); - seg = next; - } - let mut hseg = h.huge_segments; - while !hseg.is_null() { - let pg = &raw mut (*hseg).pages[1]; - page_set_flag(pg, XFLAG_NEVER); - (*pg).xheap.store(0, Ordering::Release); - let next = (*hseg).next; - let _ = segment::huge_free(hseg); - hseg = next; - } - release_heap_box(hb); - } -} - -fn merge_stats(into: &mut crate::heap::Stats, from: &crate::heap::Stats) { - into.allocs += from.allocs; - into.frees += from.frees; - into.generic += from.generic; - into.pages_fresh += from.pages_fresh; - into.segments += from.segments; - into.huge_allocs += from.huge_allocs; - into.extends += from.extends; - into.large_allocs += from.large_allocs; - into.pages_retired += from.pages_retired; - into.segments_freed += from.segments_freed; - into.realloc_in_place += from.realloc_in_place; - into.realloc_moved += from.realloc_moved; - into.delayed_frees += from.delayed_frees; - into.reclaims += from.reclaims; -} - -/// Unregister and release a HeapBox, repairing the TLS default if needed. -/// -/// # Safety -/// `hb` live, owned by the calling thread, all contents already migrated or -/// released. -unsafe fn release_heap_box(hb: *mut HeapBox) { - heaps_unregister(hb); - if heap_tls::get() == hb { - heap_tls::set(backing_heap()); - } - // A first-class heap created before the first allocation would have been - // handed the fixed backend's static descriptor; that is not a page and - // must not be returned to the region. - if crate::ONE_REGION && prim::fixed::is_first_heap_box(hb) { - return; - } - let size = core::mem::size_of::(); - let blockdesc = os::OsBlock { - ptr: hb.cast(), - size: os::page_align_up(size), - is_large: false, - is_zero: false, - }; - // SAFETY: hb's mapping came from create_heap's os::alloc_aligned with this - // exact size; unregistered above, no references remain. - unsafe { - let _ = os::free(blockdesc); - } -} - -// --------------------------------------------------------------------------- -// Global abandoned-segment list (spin-locked; cold path). -// --------------------------------------------------------------------------- - -/// Subprocess isolation (M6, plan §5.9): threads tagged with a subproc id -/// abandon/reclaim only within it — separate interpreters never exchange -/// segments. Id 0 is the main subprocess. -const MAX_SUBPROCS: usize = 64; - -static ABANDONED_LOCK: AtomicBool = AtomicBool::new(false); -static ABANDONED_HEADS: [AtomicPtr; MAX_SUBPROCS] = - [const { AtomicPtr::new(ptr::null_mut()) }; MAX_SUBPROCS]; -static SUBPROC_NEXT: AtomicUsize = AtomicUsize::new(1); -/// Diagnostic: segments currently abandoned (all subprocs). -pub static ABANDONED_COUNT: AtomicUsize = AtomicUsize::new(0); - -ra_thread_local! { - static SUBPROC: Cell = const { Cell::new(0) }; -} - -/// `mi_subproc_main` → 0. -pub fn subproc_main() -> usize { - 0 -} - -/// `mi_subproc_new`: allocate a fresh subprocess id. -/// -/// Exhaustion is an invalid id (`MAX_SUBPROCS`), not an assert (OH-rusty_alloc-21). -pub fn subproc_new() -> usize { - loop { - let id = SUBPROC_NEXT.load(Ordering::Acquire); - if id >= MAX_SUBPROCS { - return MAX_SUBPROCS; - } - if SUBPROC_NEXT - .compare_exchange_weak(id, id + 1, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - { - return id; - } - } -} - -/// `mi_subproc_delete`: ids are not recycled in v1; abandoned segments of the -/// subproc are drained into the main subproc so they stay reclaimable. -pub fn subproc_delete(id: usize) { - if id == 0 || id >= MAX_SUBPROCS { - return; - } - abandoned_lock(); - // SAFETY: list surgery under the lock. - unsafe { - let mut seg = ABANDONED_HEADS[id].swap(ptr::null_mut(), Ordering::AcqRel); - while !seg.is_null() { - let next = (*seg).next; - (*seg).next = ABANDONED_HEADS[0].load(Ordering::Relaxed); - ABANDONED_HEADS[0].store(seg, Ordering::Relaxed); - seg = next; - } - } - abandoned_unlock(); -} - -/// `mi_subproc_add_current_thread`. -pub fn subproc_add_current_thread(id: usize) { - if id >= MAX_SUBPROCS { - return; - } - SUBPROC.with(|c| c.set(id)); - // Mirror onto this thread's backing heap so TEARDOWN can read the tag - // without touching the thread-local (see `HeapBox::subproc`). Tagging - // before the first allocation is the common order and is covered by - // `create_heap` reading `my_subproc()`; this covers tagging AFTER the heap - // already exists. Deliberately does NOT bootstrap a heap — a thread that - // tags itself and never allocates should not be given one. - let hb = BACKING_PTR.with(|c| c.get()); - if !hb.is_null() { - // SAFETY: hb is this thread's own live backing box; `subproc` is - // atomic, so the teardown read needs no other synchronisation. - unsafe { (*hb).subproc.store(id, Ordering::Release) }; - } -} - -fn my_subproc() -> usize { - SUBPROC.with(|c| c.get()) -} - -/// `mi_abandoned_visit_blocks`: visit live pages/blocks of a subprocess's -/// abandoned segments, holding the list lock (segments are pinned meanwhile). -pub fn abandoned_visit_blocks( - subproc_id: usize, - heap_tag: i32, - visit_blocks: bool, - f: &mut dyn FnMut(&crate::heap::AreaInfo, *mut u8, usize) -> bool, -) -> bool { - if subproc_id >= MAX_SUBPROCS { - return false; - } - abandoned_lock(); - let mut ok = true; - let mut seg = ABANDONED_HEADS[subproc_id].load(Ordering::Acquire); - while !seg.is_null() && ok { - // SAFETY: the lock pins abandoned segments (adopt also takes it). - unsafe { - ok = crate::heap::visit_segment_blocks(seg, heap_tag, visit_blocks, false, f); - seg = (*seg).next; - } - } - abandoned_unlock(); - ok -} - -fn abandoned_lock() { - while ABANDONED_LOCK - .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) - .is_err() - { - core::hint::spin_loop(); - } -} - -fn abandoned_unlock() { - ABANDONED_LOCK.store(false, Ordering::Release); -} - -/// Publish `seg` to subprocess `sp`'s abandoned list. -/// -/// `sp` is passed in rather than read from `my_subproc()`: every caller is on -/// the teardown path, where the `SUBPROC` thread-local may already be gone (see -/// `HeapBox::subproc`). The owning heap box carries the tag instead. -fn abandoned_push(seg: *mut Segment, sp: usize) { - // The correctness argument behind `abandoned_pop`'s early return: if this - // ever runs on a single-context build, a thread ended, and the target's - // single-thread assertion was false all along. Worth knowing for its own - // sake -- `SingleThreadCell` and `prim::fixed` rest on the same claim. - if crate::ONE_THREAD { - unreachable!("a segment was abandoned on a build that asserted a single thread"); - } - debug_assert!(sp < MAX_SUBPROCS); - // PURGE BEFORE ORPHANING (`abandoned_page_purge`, on by default). Until - // some other thread adopts this segment it has no owner, so every page it - // touched stays resident for an unbounded time and nothing can reuse it. - // Done here, before publishing to the list, because this is the last - // instant the caller still exclusively owns the segment. - if crate::options::is_enabled(30) { - // SAFETY: still owned by the dying thread; free spans hold no live - // blocks, and each purged span is marked so reuse re-commits it. - unsafe { crate::segment::purge_free_spans(seg) }; - } - abandoned_lock(); - // SAFETY: seg is disowned; its `next` is ours to use as the list link. - unsafe { - (*seg).next = ABANDONED_HEADS[sp].load(Ordering::Relaxed); - } - ABANDONED_HEADS[sp].store(seg, Ordering::Relaxed); - ABANDONED_COUNT.fetch_add(1, Ordering::Relaxed); - abandoned_unlock(); -} - -/// Pop one abandoned segment from the CALLER's subprocess and take ownership. -/// Returns null when none are available. -pub fn abandoned_pop() -> *mut Segment { - // A single-context build has nothing to pop, by construction: a segment is - // only ever abandoned by a thread that ENDS (`thread_done`), and the - // target has asserted there is exactly one thread for the life of the - // program. Folding this to null is what makes `adopt_segment` unreachable - // from the allocation path, so the linker drops it -- 1,154 bytes on an - // ESP32-S3 that could never have run (`docs/plans/finished/firmware-code-size.md`, - // lever 1). `heap_delete` still calls it directly, and a firmware that - // deletes first-class heaps keeps it; that is a real path, not a - // cross-thread one. - if crate::ONE_THREAD { - return ptr::null_mut(); - } - let sp = my_subproc(); - if ABANDONED_HEADS[sp].load(Ordering::Acquire).is_null() { - return ptr::null_mut(); // fast no-lock exit for the common case - } - abandoned_lock(); - let seg = ABANDONED_HEADS[sp].load(Ordering::Relaxed); - if !seg.is_null() { - // SAFETY: list links are protected by the lock. - unsafe { - ABANDONED_HEADS[sp].store((*seg).next, Ordering::Relaxed); - (*seg).next = ptr::null_mut(); - (*seg).thread_id.store(thread_id(), Ordering::Release); - } - ABANDONED_COUNT.fetch_sub(1, Ordering::Relaxed); - } - abandoned_unlock(); - seg -} +//! Thread + heap lifecycle (mirrors upstream `init.c` + the abandonment side +//! of `arena-abandon.c`). +//! +//! Every thread gets a lazily-created [`HeapBox`] (one OS page from the prim +//! layer — never from the allocator itself, so bootstrap can't recurse). The +//! fast path reaches it through a const-init `thread_local!` `Cell` pointer, +//! which the R1 spike measured at atomic-load parity (0.3 ns) — no dtor is +//! registered on that key (Cell is !Drop), so accessing it never allocates. +//! +//! Thread EXIT runs through the prim TLS destructor (`FlsAlloc` / +//! `pthread_key_create`, built in M1 for exactly this): drain delayed frees, +//! retire what died, transition every surviving page to `NEVER` (spinning out +//! in-flight `FREEING` remotes — the loom-modeled teardown order), zero the +//! segments' thread id, and publish them on the global abandoned list. Any +//! other thread's allocation slow path adopts them later. + +use core::cell::{Cell, UnsafeCell}; +use core::ffi::c_void; +use core::ptr; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +use crate::heap::Heap; +use crate::page::{ + DelayedList, XFLAG_NEVER, page_collect, page_collect_and_set_flag, page_set_flag, +}; +use crate::segment::{self, Segment}; +use crate::{os, prim}; + +/// A heap plus the cross-thread fields remote threads may touch. Remote +/// threads only ever dereference `delayed` (via page `xheap` addresses) — +/// never the heap itself — which is what makes `&mut` on the inner heap sound. +#[repr(C)] +pub struct HeapBox { + /// Cross-thread delayed-free list (remote-reachable). MUST stay the first + /// field: pages store `&delayed` in `xheap`, and owner routing recovers + /// the box by container-of (delayed at offset 0 ⇒ the addresses coincide). + pub delayed: DelayedList, + /// Next box in the global registry (guarded by the registry lock). + /// AtomicPtr, not usize: reachability (miri's leak check and honesty + /// about what is cached vs leaked) follows pointers, not integers. + pub next_box: AtomicPtr, + /// Owning thread id (heaps allocate only from their creating thread). + pub owner_tid: usize, + /// The subprocess this heap's segments are abandoned into. + /// + /// Mirrored from the `SUBPROC` thread-local so that TEARDOWN never has to + /// read it. `thread_done` runs from a platform TLS destructor + /// (`pthread_key_create` / `FlsAlloc`), and the destruction order between + /// *that* callback and Rust's own `thread_local!` storage is unspecified on + /// every platform. On aarch64-apple-darwin it is observably wrong: + /// `my_subproc()` reads back 0 inside `thread_done`, so a tagged thread's + /// segments landed in the MAIN subprocess and `mi_subproc_*` isolation was + /// silently lost — the abandon fired correctly, only its destination was + /// wrong. The heap box is our own mapping and is alive for the whole of + /// teardown (it is the value passed *to* the destructor), so reading the tag + /// from here is correct on every target instead of accidentally correct on + /// one. + pub subproc: AtomicUsize, + /// Heap tag (`mi_heap_new_ex`; reported by the block visitor). + pub tag: i32, + /// Whether `mi_heap_destroy` may drop live blocks (else it delegates to + /// delete semantics). + pub allow_destroy: bool, + /// The owner-only heap state. + pub heap: UnsafeCell, +} + +const _: () = assert!(core::mem::offset_of!(HeapBox, delayed) == 0); + +/// Wrapper so the empty sentinel box can be a `static`. +#[repr(transparent)] +pub struct EmptyHeapBox(HeapBox); + +// SAFETY: the sentinel is never written and never escapes to remote threads: +// no page ever stores its `delayed` address in `xheap` (pages are re-homed +// only by REAL heaps), its `owner_tid` of 0 matches no live thread, and every +// reader goes through raw-pointer READS — `heap_box_fast` documents that no +// `&mut` may ever be formed to it. It exists purely so the malloc fast path +// has a valid heap to read before this thread has created one. +unsafe impl Sync for EmptyHeapBox {} + +/// The "no heap yet" sentinel: every `direct` slot holds the empty PAGE +/// sentinel, so a fresh thread's first malloc falls through to the generic +/// path — which detects this box and performs the real thread init — without +/// the fast path ever testing for it. This is upstream's `_mi_heap_empty` +/// trick, and the exact design our own [`crate::page::EMPTY_PAGE`] already +/// uses one level down: replace a null-test on the hot path with a sentinel +/// whose data routes the miss to the slow path for free. +/// +/// `export_name` (not mangled) because the x86-64 Linux TLS slot's `.tdata` +/// initializer below names it in assembly; `.hidden` there keeps it out of +/// the cdylib's export table. +/// +/// **In flash on a one-region target.** The sentinel is read and never +/// written — `heap_box_fast` documents that contract, the fast path only +/// writes after a non-null pop and every `direct` slot here points at the +/// empty page whose free list is null, and no page ever carries its address +/// in `xheap`. Its type has interior mutability (a `DelayedList`, an +/// `AtomicPtr`, the `UnsafeCell`), which is why rustc places it in +/// `.data` by default: 1,752 bytes paid once in flash for the image and once +/// more in RAM for the loader's copy, the largest single item left on the +/// ESP32-S3 after 2.0.2 and RAM the linker takes from the stack. Placing it +/// in `.rodata.*` keeps the flash copy and drops the RAM one. Any write +/// would fault loudly against the flash cache, which is the right way for +/// the contract to fail. Bare metal only — a hosted target may need to +/// relocate the pointers inside it at load, and has RAM to spare. +#[cfg_attr( + all( + ra_single_threaded, + not(miri), + not(windows), + not(unix), + not(target_arch = "wasm32") + ), + unsafe(link_section = ".rodata.rusty_alloc_empty_heap_box") +)] +#[unsafe(export_name = "__ra_empty_heap_box")] +pub static EMPTY_HEAP_BOX: EmptyHeapBox = EmptyHeapBox(HeapBox { + delayed: DelayedList::new(), + next_box: AtomicPtr::new(ptr::null_mut()), + owner_tid: 0, + subproc: AtomicUsize::new(0), + tag: 0, + allow_destroy: false, + heap: UnsafeCell::new(Heap::new()), +}); + +/// Pointer to the shared empty heap box. +#[inline] +pub const fn empty_heap_box_ptr() -> *mut HeapBox { + (&raw const EMPTY_HEAP_BOX.0).cast_mut() +} + +/// Recover the owning HeapBox from a page's `xheap` value (the container-of +/// trick over the offset-0 delayed list). +/// +/// # Safety +/// `xheap` must be a live HeapBox's delayed-list address (nonzero). +#[inline] +pub unsafe fn box_of_xheap(xheap: usize) -> *mut HeapBox { + debug_assert!(xheap != 0); + xheap as *mut HeapBox +} + +// Global registry of live heap boxes: keeps every heap (and through it every +// segment) reachable-from-static — the process-wide stats walk of M7, and +// what makes miri's reachability-based leak check reflect reality (an +// allocator's arenas at process exit are cached state, not leaks). +static HEAPS_LOCK: AtomicBool = AtomicBool::new(false); +static HEAPS_HEAD: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +fn heaps_lock() { + while HEAPS_LOCK + .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + core::hint::spin_loop(); + } +} + +fn heaps_unlock() { + HEAPS_LOCK.store(false, Ordering::Release); +} + +fn heaps_register(hb: *mut HeapBox) { + heaps_lock(); + // SAFETY: hb is fully initialized and not yet published. + unsafe { + (*hb) + .next_box + .store(HEAPS_HEAD.load(Ordering::Relaxed), Ordering::Relaxed); + } + HEAPS_HEAD.store(hb, Ordering::Relaxed); + heaps_unlock(); +} + +fn heaps_unregister(hb: *mut HeapBox) { + heaps_lock(); + let mut cur = HEAPS_HEAD.load(Ordering::Relaxed); + if cur == hb { + // SAFETY: links guarded by the lock. + unsafe { + HEAPS_HEAD.store((*hb).next_box.load(Ordering::Relaxed), Ordering::Relaxed); + } + } else { + while !cur.is_null() { + // SAFETY: links guarded by the lock; nodes are live boxes. + unsafe { + let nxt = (*cur).next_box.load(Ordering::Relaxed); + if nxt == hb { + (*cur) + .next_box + .store((*nxt).next_box.load(Ordering::Relaxed), Ordering::Relaxed); + break; + } + cur = nxt; + } + } + } + heaps_unlock(); +} + +/// Unlink and return one live heap box owned by `tid`, or null. +/// +/// The lock is released before return so the caller can abandon the heap +/// without re-entering `HEAPS_LOCK` (abandon walks pages and can allocate). +fn take_heap_owned_by(tid: usize) -> *mut HeapBox { + heaps_lock(); + let mut prev: *mut HeapBox = ptr::null_mut(); + let mut cur = HEAPS_HEAD.load(Ordering::Relaxed); + while !cur.is_null() { + // SAFETY: links and boxes are live under the lock. + unsafe { + let nxt = (*cur).next_box.load(Ordering::Relaxed); + if (*cur).owner_tid == tid { + if prev.is_null() { + HEAPS_HEAD.store(nxt, Ordering::Relaxed); + } else { + (*prev).next_box.store(nxt, Ordering::Relaxed); + } + heaps_unlock(); + return cur; + } + prev = cur; + cur = nxt; + } + } + heaps_unlock(); + ptr::null_mut() +} + +/// Storage for the calling thread's default-heap pointer. +/// +/// On x86-64 Linux this is a RAW ELF TLS symbol reached with the +/// **initial-exec** model rather than `thread_local!`'s general-dynamic. That +/// was the last `__tls_get_addr` call on the malloc fast path — 7.3 M +/// instructions on perl, a third of everything still separating us from +/// mimalloc after the M10b free-path bricks. Stable Rust has no +/// `-Z tls-model`, so the symbol is declared in `global_asm!` and the linker +/// resolves its offset. +/// +/// **Why this is sound where a thread-pointer-keyed side table is not.** The +/// storage IS the thread's own TLS block. Every thread gets a fresh block +/// initialised from the `.tdata` template (the empty-heap sentinel) when it +/// is created, so a recycled TCB cannot hand a new thread a dead thread's +/// heap — the staleness +/// question that made the keyed-cache design a P0 risk simply does not arise +/// here. What initial-exec costs instead is a LOAD-TIME constraint: it needs a +/// slot in the static TLS block, so a `dlopen` long after startup could fail +/// to load us. That fails loudly at load time rather than corrupting memory, +/// and it is the same trade upstream mimalloc ships +/// (`__attribute__((tls_model("initial-exec")))` on `_mi_heap_default`). +#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] +mod heap_tls { + use super::HeapBox; + + // An 8-byte thread-local slot. `.tdata` (not `.tbss`): the slot is + // INITIALISED to the empty-heap sentinel, so every new thread reads a + // valid heap pointer from its very first malloc and the fast path needs + // no "not yet created" test at all — the sentinel's empty direct table + // routes the first allocation to the generic path, which performs the + // real init there (upstream's `_mi_heap_empty` design). The loader + // relocates the `.quad` in the TLS template once at load time; every + // thread's block is then copied from that template. `.hidden` makes both + // symbols non-preemptible, which is what permits the initial-exec + // (GOTTPOFF) relocation instead of a general-dynamic call, and keeps the + // sentinel out of the cdylib's export table. + core::arch::global_asm!( + ".section .tdata,\"awT\",@progbits", + ".globl __ra_tls_heap", + ".hidden __ra_tls_heap", + ".hidden __ra_empty_heap_box", + ".type __ra_tls_heap,@object", + ".p2align 3", + "__ra_tls_heap:", + ".quad __ra_empty_heap_box", + ".size __ra_tls_heap,8", + ".text", + ); + + /// Address of THIS thread's slot: the thread pointer plus the offset the + /// dynamic linker wrote into the GOT. + #[inline(always)] + fn slot() -> *mut *mut HeapBox { + let off: usize; + // SAFETY: reads the GOT entry the dynamic linker relocated at load + // time, before any of our code can run, and never writes. It is + // constant for the life of the process, hence `pure` + `readonly` — + // that is what lets LLVM hoist and CSE the address computation + // instead of treating the block as an optimisation barrier. + unsafe { + core::arch::asm!( + "mov {o}, qword ptr [rip + __ra_tls_heap@GOTTPOFF]", + o = out(reg) off, + options(nostack, preserves_flags, readonly, pure), + ); + } + // `thread_id()` IS the fs base on this target. TLS offsets are + // negative there (the block grows down from the thread pointer), so + // this adds a two's-complement offset and must wrap. + core::ptr::with_exposed_provenance_mut(super::thread_id().wrapping_add(off)) + } + + /// Read this thread's heap pointer in TWO instructions. + /// + /// The obvious form — `slot().read()` — costs FOUR: load the offset from + /// the GOT, read the thread pointer from `fs:0`, add them, dereference. + /// But x86 performs that addition IN THE ADDRESSING MODE: `fs:[reg]` is a + /// segment-relative load, so the explicit `fs:0` read and the add both + /// disappear. Measured at 4.00 Ir per malloc before this change — a + /// quarter of our entire malloc-side deficit spent just locating the heap. + #[inline(always)] + pub fn get() -> *mut HeapBox { + let v: *mut HeapBox; + // SAFETY: reads the GOT entry the loader relocated at startup, then + // loads 8 aligned bytes of this thread's own TLS block through the fs + // segment. Both are reads; nothing is written. + unsafe { + core::arch::asm!( + "mov {t}, qword ptr [rip + __ra_tls_heap@GOTTPOFF]", + "mov {o}, qword ptr fs:[{t}]", + t = out(reg) _, + o = out(reg) v, + options(nostack, preserves_flags, readonly), + ); + } + v + } + + #[inline(always)] + pub fn set(hb: *mut HeapBox) { + // SAFETY: as `get` — the slot is owned exclusively by this thread. + unsafe { slot().write(hb) } + } +} + +/// Every other target keeps the `thread_local!`: Windows TLS is an index into +/// a per-thread array (no `__tls_get_addr` to remove), and miri models the +/// macro but not a hand-rolled TLS symbol. +#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))] +mod heap_tls { + use super::HeapBox; + use core::cell::Cell; + + ra_thread_local! { + /// Fast-path heap pointer. Const-init + !Drop ⇒ plain TLS access, no + /// lazy-init branch, no allocation ever. Initialised to the + /// empty-heap SENTINEL (never null) so the malloc fast path can read + /// through it unconditionally; see `EMPTY_HEAP_BOX`. + static HEAP_PTR: Cell<*mut HeapBox> = const { Cell::new(super::empty_heap_box_ptr()) }; + } + + #[inline(always)] + pub fn get() -> *mut HeapBox { + HEAP_PTR.with(|c| c.get()) + } + + #[inline(always)] + pub fn set(hb: *mut HeapBox) { + HEAP_PTR.with(|c| c.set(hb)); + } +} + +ra_thread_local! { + /// Cached OS thread id. `free` needs the calling thread's id on EVERY + /// call to route local-vs-remote; the raw `prim::thread_id()` is a libc + /// call (`pthread_self` through the PLT from a cdylib) measured at + /// 1.41 ns vs 0.25 ns for this cache — ~18-20% of a whole malloc+free + /// pair (M9 probe, both platforms). 0 = not yet cached; no real thread id + /// is 0 on either platform. + static TID: Cell = const { Cell::new(0) }; +} + +/// The calling thread's id — read STRAIGHT FROM THE THREAD-POINTER REGISTER +/// where the target allows it. +/// +/// Why this and not `thread_local!`: the shipping artifact is a **cdylib** +/// (LD_PRELOAD). Rust's `thread_local!` in a shared library compiles to the +/// general-dynamic TLS model, so every access is a CALL into `ld.so`'s +/// `__tls_get_addr` — the per-function instruction profile put that call at +/// 12.97 M Ir, 1.96% of a whole lua run, comparable to our entire `free`. +/// (The earlier ns-level probe missed it because it measured TLS inside an +/// EXECUTABLE, where the model is local-exec: a register offset, no call. +/// Measure the artifact you ship.) +/// +/// A ceiling probe with `-Z tls-model=initial-exec` valued removing those +/// calls at 2.00% of our instructions and 33% of the gap to mimalloc; this +/// gets the same effect on stable, for the id, with no linker involvement — +/// the thread pointer is one register read. It is exactly what mimalloc's +/// `_mi_thread_id()` does. +/// +/// Uniqueness: the value is the thread-control-block address, unique among +/// LIVE threads. It can be recycled after a thread dies, which is sound here +/// for the same reason it is in mimalloc: a dying thread abandons its +/// segments (`thread_id` stored as 0) before its TCB can be reused, so a +/// recycled id can never match a stale segment. +#[inline(always)] +pub fn thread_id() -> usize { + #[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] + { + let tp: usize; + // SAFETY: reads the TCB self-pointer at fs:0, which the x86-64 Linux + // ABI guarantees is present and thread-unique. No memory is written. + unsafe { + core::arch::asm!("mov {}, fs:0", out(reg) tp, + options(nostack, preserves_flags, readonly)); + } + tp + } + #[cfg(all(target_arch = "x86_64", target_os = "windows", not(miri)))] + { + let teb: usize; + // SAFETY: gs:0x30 is the TEB self-pointer on x86-64 Windows — + // present in every thread, unique, read-only here. + unsafe { + core::arch::asm!("mov {}, gs:0x30", out(reg) teb, + options(nostack, preserves_flags, readonly)); + } + teb + } + // AArch64 splits by OS: the register holding the thread pointer is NOT the + // same one on Darwin as everywhere else. + #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple"), not(miri)))] + { + let tp: usize; + // SAFETY: on the standard AArch64 ABI (Linux, Android, BSD) tpidr_el0 + // IS the thread pointer; read-only. + unsafe { + core::arch::asm!("mrs {}, tpidr_el0", out(reg) tp, + options(nostack, preserves_flags, readonly)); + } + tp + } + #[cfg(all(target_arch = "aarch64", target_vendor = "apple", not(miri)))] + { + let tp: usize; + // SAFETY: read-only system-register read. + // + // DARWIN IS NOT THE STANDARD AArch64 ABI. Apple puts the thread pointer + // in tpidrRO_el0 and uses tpidr_el0 for the CPU/cluster id. Reading + // tpidr_el0 here — as every other AArch64 target correctly does — was a + // memory-safety bug, measured on macOS 26 / M-series: + // + // * tpidr_el0 returns small values (0x1002, 0x2005, …), not pointers; + // * it CHANGES under a single thread as that thread migrates between + // cores (5 distinct values over 3M reads on the main thread); + // * and 8 live threads produced only 5 distinct values — DIFFERENT + // THREADS COLLIDED. + // + // thread_id() is the allocator's ownership identity: `segment.thread_id` + // decides whether a free takes the local (unsynchronised) path or the + // remote one. Colliding ids let one thread free into another thread's + // segment through the owner path, racing non-atomic page state — heap + // corruption, not merely a lost optimisation. A drifting id separately + // makes a thread stop recognising its own segments. + // + // tpidrro_el0 is the real thread pointer: stable within a thread and + // unique across live threads (it is `pthread_self() + 0xe0`). Apple + // documents the LOW 3 BITS as the current CPU number, so they must be + // masked off — that is precisely the drift observed above. + unsafe { + core::arch::asm!("mrs {}, tpidrro_el0", out(reg) tp, + options(nostack, preserves_flags, readonly)); + } + tp & !0b111 + } + // Every other target: the cached-TLS path (still far cheaper than asking + // the OS on each call). + #[cfg(not(any( + all(target_arch = "x86_64", target_os = "linux", not(miri)), + all(target_arch = "x86_64", target_os = "windows", not(miri)), + all(target_arch = "aarch64", not(miri)) + )))] + { + let t = TID.with(|c| c.get()); + if t != 0 { t } else { init_tid() } + } +} + +#[cold] +#[cfg_attr( + any( + all(target_arch = "x86_64", target_os = "linux", not(miri)), + all(target_arch = "x86_64", target_os = "windows", not(miri)), + all(target_arch = "aarch64", not(miri)) + ), + allow(dead_code) +)] +fn init_tid() -> usize { + let t = prim::thread_id(); + TID.with(|c| c.set(t)); + t +} + +/// Snapshot-visit every registered heap (stats aggregation). The visitor +/// receives a COPY of the heap's counters. +/// +/// **This read is racy by design and `volatile` does not make it less so** — +/// clippy's `volatile_composites` is right to flag it, and the lint is +/// allowed HERE rather than workspace-wide so the reasoning stays attached to +/// the one site that does it. `read_volatile` on a composite gives no +/// per-field atomicity and no ordering, so a concurrent owner mutating its +/// counters can yield a snapshot whose fields disagree with each other. That +/// is accepted: these are `#[cfg(debug_assertions)]` diagnostic counters +/// (upstream's `MI_STAT` rule), nothing branches on them, and the alternative +/// — taking `&` to a struct another thread is writing — would be UB rather +/// than merely imprecise. The real fix is making the counters atomics, which +/// is the tracked follow-up (R-005 in the hardening audit). +#[allow( + clippy::volatile_composites, + reason = "racy diagnostic snapshot; see the doc comment. Tracked as R-005." +)] +pub fn for_each_heap(f: &mut dyn FnMut(&crate::heap::Heap)) { + heaps_lock(); + let mut hb = HEAPS_HEAD.load(Ordering::Acquire); + while !hb.is_null() { + // SAFETY: registry entries are live boxes. The read is unsynchronised + // with the owning thread's writes (see above) but stays within a live, + // correctly-aligned allocation for the whole of the copy. + unsafe { + let snapshot = core::ptr::read_volatile((*hb).heap.get()); + f(&snapshot); + hb = (*hb).next_box.load(Ordering::Acquire); + } + } + heaps_unlock(); +} + +/// Get (or create) the calling thread's heap box. NULL when creation fails +/// (memory exhaustion) — callers translate that into their own failure value +/// (a null allocation, a no-op collect); the TLS slot keeps the sentinel so a +/// later call retries. +#[inline] +pub fn heap_box() -> *mut HeapBox { + let hb = heap_tls::get(); + if hb == empty_heap_box_ptr() { + init_thread_heap() + } else { + hb + } +} + +/// This thread's heap box WITHOUT the created-yet test — may return the +/// shared empty sentinel. For the malloc fast path only. +/// +/// # Safety (for callers) +/// The returned box may be [`EMPTY_HEAP_BOX`], which is shared between every +/// uninitialised thread: callers must perform RAW-POINTER READS ONLY and must +/// never form a `&mut` (or write) through it. Route any miss to +/// [`ensure_heap`] before mutating anything. +#[inline] +pub fn heap_box_fast() -> *mut HeapBox { + heap_tls::get() +} + +/// Upgrade a possibly-sentinel box from [`heap_box_fast`] to this thread's +/// real, initialised heap box. NULL when creation fails (see [`heap_box`]). +#[inline] +pub fn ensure_heap(hb: *mut HeapBox) -> *mut HeapBox { + if hb == empty_heap_box_ptr() { + init_thread_heap() + } else { + hb + } +} + +/// Allocate + initialize a HeapBox (shared by thread bootstrap and +/// `mi_heap_new*`). Registered in the global registry; NOT installed in TLS. +/// +/// Returns NULL when the OS refuses the mapping (memory exhaustion). This +/// used to `expect()` — which, under the release profile's `panic = "abort"`, +/// turned OOM into a process abort on a path reachable from any fresh +/// thread's first `malloc`. The malloc contract on OOM is a null return, and +/// upstream returns NULL from `mi_heap_new` for the same reason (H-18/R-003 +/// in the hardening audit). +pub fn create_heap(tag: i32, allow_destroy: bool, arena_id: i32) -> *mut HeapBox { + // First-class heaps are not stored in `done_slot`. A thread that only + // ever called `mi_heap_new` / `heap_malloc` never ran `init_thread_heap`, + // so the TLS destructor never fired and leftover heaps stayed DELAYED + // (OH-rusty_alloc-13). Bootstrap the backing heap (and the exit hook) + // first. `init_thread_heap` calls `create_heap_uninstalled`, not this + // function, so there is no recursion. + if !crate::ONE_THREAD && heap_tls::get() == empty_heap_box_ptr() { + let _ = heap_box(); + } + create_heap_uninstalled(tag, allow_destroy, arena_id) +} + +/// Released heap boxes kept for the next heap, as upstream keeps +/// `mi_thread_data_t` (`TD_CACHE_SIZE`). A thread pool spawns and retires +/// threads constantly; without this, every thread's heap box was a fresh +/// `mmap` at its first allocation and a `munmap` at its exit, plus the +/// first-touch fault on the new mapping. A box in the cache is unregistered +/// and referenced by nothing (exactly the state in which it used to be +/// unmapped), and `create_heap_uninstalled` rewrites the whole struct. +static HEAP_BOX_CACHE: [AtomicPtr; 8] = [const { AtomicPtr::new(ptr::null_mut()) }; 8]; + +/// A cached heap box, or null. +fn heap_box_cache_take() -> *mut HeapBox { + for slot in &HEAP_BOX_CACHE { + if !slot.load(Ordering::Relaxed).is_null() { + let hb = slot.swap(ptr::null_mut(), Ordering::Acquire); + if !hb.is_null() { + return hb; + } + } + } + ptr::null_mut() +} + +/// Park a released heap box for reuse; false when the cache is full. +fn heap_box_cache_put(hb: *mut HeapBox) -> bool { + for slot in &HEAP_BOX_CACHE { + if slot + .compare_exchange(ptr::null_mut(), hb, Ordering::Release, Ordering::Relaxed) + .is_ok() + { + return true; + } + } + false +} + +fn create_heap_uninstalled(tag: i32, allow_destroy: bool, arena_id: i32) -> *mut HeapBox { + let size = core::mem::size_of::(); + // On a one-region target the FIRST descriptor is a static of the fixed + // backend's, so that a firmware's region is whole segments and needs no + // page carved out of it — the page is what made an aligned container of + // the region pad itself out to the next segment + // (`docs/plans/finished/region-alignment-bug.md` §7). Every later heap, + // and every heap on a hosted target, takes a page as before. + let first = if crate::ONE_REGION { + prim::fixed::take_first_heap_box() + } else { + None + }; + let hb: *mut HeapBox = match first { + Some(p) => p, + None => { + let cached = heap_box_cache_take(); + if cached.is_null() { + let Ok(block) = os::alloc_aligned(size, os::page_size(), true, false) else { + return ptr::null_mut(); + }; + block.ptr.cast() + } else { + cached + } + } + }; + // SAFETY: fresh committed mapping large enough for HeapBox; we initialize + // every field before publishing the pointer. + unsafe { + ptr::write( + hb, + HeapBox { + delayed: DelayedList::new(), + next_box: AtomicPtr::new(ptr::null_mut()), + owner_tid: thread_id(), + // Captured HERE, on the live creating thread, where the + // thread-local is unambiguously valid. + subproc: AtomicUsize::new(my_subproc()), + tag, + allow_destroy, + // A new heap starts as a copy of the empty one. On a + // one-region target that copy is taken from the sentinel + // itself, which already holds the whole template in flash; + // materialising `Heap::new()` here as well cost a second + // ~900-byte `.rodata` blob for the bin table. Hosted builds + // keep the constant, whose codegen was measured in Ir. + heap: UnsafeCell::new(if crate::ONE_REGION { + // SAFETY (within the enclosing block): a raw, + // non-overlapping read of the never-written sentinel's + // heap — the only access `heap_box_fast`'s contract + // permits, and `Heap` owns nothing that a bitwise copy + // could double-free. + ptr::read((*empty_heap_box_ptr()).heap.get()) + } else { + Heap::new() + }), + }, + ); + (*(*hb).heap.get()).delayed = &raw const (*hb).delayed; + (*(*hb).heap.get()).arena_id = arena_id; + (*(*hb).heap.get()).tag = tag; + // Per-heap CSPRNG stream (free-list keys, guarded sampling) -- seeded + // only where something draws from it. A build with neither `secure` + // nor guard pages has no consumer, and seeding still cost the ChaCha + // block function plus the splitmix fallback in flash. + if crate::RNG_USED { + (*(*hb).heap.get()).rng.reseed(); + } + // Guarded objects follow the option table (MI_GUARDED-equivalent) -- + // where a guard page can exist at all. `GUARD_PAGES` first, so that on + // a target without one the option reads fold away with the sampler + // (`docs/plans/finished/firmware-code-size.md`, lever 3). + let rate = crate::options::get(33).max(0) as usize; // guarded_sample_rate + let gmin = crate::options::get(30).max(0) as usize; // guarded_min + let gmax = crate::options::get(31).max(0) as usize; // guarded_max + if crate::GUARD_PAGES && gmax > 0 { + (*(*hb).heap.get()).guarded_set_size_bound(gmin, gmax); + (*(*hb).heap.get()) + .guarded_set_sample_rate(rate, crate::options::get(34).max(0) as usize); + } + } + heaps_register(hb); + hb +} + +#[cold] +fn init_thread_heap() -> *mut HeapBox { + let hb = create_heap_uninstalled(0, false, -1); + if hb.is_null() { + // OOM: install nothing — the TLS slot keeps the sentinel, so the + // NEXT allocation retries creation instead of dereferencing null. + return ptr::null_mut(); + } + heap_tls::set(hb); + BACKING_PTR.with(|c| c.set(hb)); + // The exit hook exists to abandon this heap when its thread ENDS. On a + // single-context build no thread ends, the backend's TLS never runs a + // destructor anyway (`prim::fixed::tls_new` discards it), and creating + // the slot was a compare-and-swap, a spin and a table entry paid once for + // nothing (`docs/plans/finished/firmware-code-size.md`, lever 2). + if !crate::ONE_THREAD { + done_slot().set(hb.cast::()); + } + hb +} + +ra_thread_local! { + /// The thread's original (backing) heap — what `mi_heap_get_backing` + /// returns regardless of `mi_heap_set_default` swaps. + static BACKING_PTR: Cell<*mut HeapBox> = const { Cell::new(ptr::null_mut()) }; +} + +/// `mi_heap_get_backing`. +pub fn backing_heap() -> *mut HeapBox { + let b = BACKING_PTR.with(|c| c.get()); + if b.is_null() { + let _ = heap_box(); // bootstraps and sets BACKING_PTR + BACKING_PTR.with(|c| c.get()) + } else { + b + } +} + +/// `mi_heap_set_default`: install `hb` as this thread's default; returns the +/// previous default. +/// +/// # Safety +/// `hb` must be a live heap owned by the calling thread. +pub unsafe fn set_default_heap(hb: *mut HeapBox) -> *mut HeapBox { + let prev = heap_box(); + heap_tls::set(hb); + prev +} + +/// The process-wide TLS slot whose destructor abandons a dying thread's heap. +fn done_slot() -> prim::TlsSlot { + static RAW: AtomicUsize = AtomicUsize::new(0); + static INIT: AtomicBool = AtomicBool::new(false); + let raw = RAW.load(Ordering::Acquire); + if raw != 0 { + // SAFETY: RAW holds an into_raw of a slot created below. + return unsafe { prim::TlsSlot::from_raw(raw - 1) }; + } + if INIT + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + // A panic here would be wrong twice over. An allocator must not unwind + // into its C callers (which is why the release profile is + // panic=abort), AND the winner of this race is the only thread that + // ever publishes RAW — so unwinding past the store below would strand + // every other thread in the spin loop FOREVER, turning a rare + // resource failure into a silent process-wide hang. Fail loudly and + // deterministically instead, identically in debug and release. + let Some(slot) = prim::TlsSlot::new(Some(thread_done_cb)) else { + crate::abort(); + }; + RAW.store(slot.into_raw() + 1, Ordering::Release); // +1: 0 is the unset sentinel + } + loop { + let raw = RAW.load(Ordering::Acquire); + if raw != 0 { + // SAFETY: as above. + return unsafe { prim::TlsSlot::from_raw(raw - 1) }; + } + core::hint::spin_loop(); + } +} + +#[cfg(all(windows, not(miri)))] +unsafe extern "system" fn thread_done_cb(v: *const c_void) { + if !v.is_null() { + // SAFETY: v is the HeapBox this thread stored in the done slot. + unsafe { thread_done(v.cast_mut().cast()) }; + } +} + +#[cfg(any(not(windows), miri))] +unsafe extern "C" fn thread_done_cb(v: *mut c_void) { + if !v.is_null() { + // SAFETY: v is the HeapBox this thread stored in the done slot. + unsafe { thread_done(v.cast()) }; + } +} + +/// Abandon the dying thread's heap. Teardown order is the loom-modeled one: +/// (1) every surviving page → NEVER (spins out FREEING remotes), (2) THEN +/// drain the delayed list, (3) THEN release the heap storage. +/// +/// # Safety +/// Must run on the dying thread with `hb` its live heap box, exactly once. +pub unsafe fn thread_done(hb: *mut HeapBox) { + // SAFETY: per contract `hb` is this thread's live box; `owner_tid` is a + // plain field. + let tid = unsafe { (*hb).owner_tid }; + // SAFETY: per contract we are the owner and this runs exactly once; + // sequencing per the model. + unsafe { thread_done_one(hb) }; + // Back to the SENTINEL, not null — the slot is never null, so a + // post-teardown malloc re-enters the generic path and re-initialises. + heap_tls::set(empty_heap_box_ptr()); + // First-class heaps this thread created are not stored in `done_slot` + // (`set_default_heap` does not update it). Leaving them DELAYED meant a + // second remote free after thread exit returned (OH-rusty_alloc-13). + // Abandon every remaining heap we still own so pages go NEVER. Iterative: + // a dying thread can own many first-class heaps; do not recurse. + if !crate::ONE_THREAD { + // Stall guard: a dying thread that keeps re-initialising (empty TLS + // malloc) must not loop forever. Same order as `collect_inner`'s + // `MAX_RECLAIM`. + const MAX_OWNED_HEAPS: usize = 1 << 20; + for _ in 0..MAX_OWNED_HEAPS { + let extra = take_heap_owned_by(tid); + if extra.is_null() { + break; + } + // SAFETY: extra is a live box this dying thread still owns. + unsafe { thread_done_one(extra) }; + } + } +} + +/// Abandon one heap box (NEVER + delayed drain + unregister + free the box). +/// +/// # Safety +/// `hb` is a live heap box owned by the calling thread, not used again. +unsafe fn thread_done_one(hb: *mut HeapBox) { + // SAFETY: per contract we are the owner; sequencing per the model. + unsafe { + // Read the subprocess tag from the BOX, not from `my_subproc()`. We are + // inside a platform TLS destructor and Rust's `thread_local!` storage + // may already be torn down — on aarch64-apple-darwin it reads back 0, + // which silently dropped every tagged thread's segments into the main + // subprocess. Clamped because a corrupt tag must not index out of + // bounds on the teardown path. + let sp = (*hb).subproc.load(Ordering::Acquire).min(MAX_SUBPROCS - 1); + let h = &mut *(*hb).heap.get(); + // Retire everything that is already dead (also drains delayed once). + // TEARDOWN variant: must NOT adopt orphans into a heap we are about to + // destroy — that was the 0.3.1 segfault. + h.collect_for_teardown(); + // Walk remaining segments: transition pages to NEVER, disown, publish. + let mut seg = h.segments; + h.segments = ptr::null_mut(); + while !seg.is_null() { + let next = (*seg).next; + let end = (*seg).next_free_slice as usize; + let mut idx = segment::HEADER_SLICES; + // Running pointer, not `pages[idx]`: `Page` is not a power-of-two + // size, so indexing costs a multiply plus a bounds check on every + // carved slice of every segment a dying thread owns. + let base: *mut crate::page::Page = (&raw mut (*seg).pages).cast(); + let mut slot: *mut crate::page::Page = base.add(idx); + while idx < end { + let len = ((*slot).slice_count as usize).max(1); + if (*slot).block_size > 0 { + // ONE locked read-modify-write, not two: taking the + // cross-thread chain and publishing NEVER are the same + // word, and doing them separately paid two CAS loops per + // live page of every segment a dying thread owns. + page_collect_and_set_flag(slot, XFLAG_NEVER); + (*slot).xheap.store(0, Ordering::Release); + } + idx += len; + // `wrapping_add`, not `add`: the pointer is only ever + // DEREFERENCED while `idx < end` holds, which the span-partition + // invariant makes in-bounds — but the final advance of the loop + // computes a pointer that may be one past the carved region, and + // forming that with `add` would be UB even unread. + slot = slot.wrapping_add(len); + } + if (*seg).used_pages == 0 { + let _ = segment::segment_free(seg); + h.stats.segments_freed += 1; + } else { + (*seg).thread_id.store(0, Ordering::Release); + abandoned_push(seg, sp); + } + seg = next; + } + // Huge segments: same teardown shape (collect NEVER-bound frees, + // release dead ones, abandon the rest). + let mut hseg = h.huge_segments; + h.huge_segments = ptr::null_mut(); + while !hseg.is_null() { + let next = (*hseg).next; + let pg = &raw mut (*hseg).pages[1]; + page_collect(pg); + if (*pg).used == 0 { + let _ = segment::huge_free(hseg); + h.stats.segments_freed += 1; + } else { + page_set_flag(pg, XFLAG_NEVER); + (*pg).xheap.store(0, Ordering::Release); + (*hseg).thread_id.store(0, Ordering::Release); + abandoned_push(hseg, sp); + } + hseg = next; + } + // Post-NEVER drain: blocks a FREEING remote landed while we walked. + // Their pages are abandoned now, so free_local would be wrong — but + // these blocks belong to pages we JUST abandoned; push them back on + // their pages' own lists (flag NEVER routes them there). + let mut b = (*hb).delayed.head.swap(0, Ordering::AcqRel) as *mut crate::page::Block; + while !b.is_null() { + let next = (*b).next; + let seg = segment::segment_of(b.cast()); + let pg = segment::page_of(seg, b.cast()); + crate::page::remote_free(pg, b); + b = next; + } + // Release the heap storage and clear the fast-path pointer. The + // first descriptor on a one-region target is a static, not a page: + // nothing to return (and this path is unreachable there anyway). + heaps_unregister(hb); + // A static first box is not a page and is never cached; any other + // box is cached when there is room, and unmapped otherwise. + let kept = + (crate::ONE_REGION && prim::fixed::is_first_heap_box(hb)) || heap_box_cache_put(hb); + if !kept { + let size = core::mem::size_of::(); + let blockdesc = os::OsBlock { + ptr: hb.cast(), + size: os::page_align_up(size), + is_large: false, + is_zero: false, + }; + let _ = os::free(blockdesc); + } + } +} + +/// `mi_heap_delete`: free the heap structure, MIGRATING its live pages and +/// segments into the thread's backing heap (blocks stay valid). +/// +/// # Safety +/// `hb` must be a live first-class heap owned by the calling thread, not the +/// backing heap, used never again. +pub unsafe fn heap_delete(hb: *mut HeapBox) { + let backing = backing_heap(); + if hb == backing { + return; // deleting the backing heap is a no-op (upstream-compatible) + } + // SAFETY: owner thread per contract; absorb everything into backing. + unsafe { + debug_assert_eq!((*hb).owner_tid, thread_id()); + let h = &mut *(*hb).heap.get(); + let bh = &mut *(*backing).heap.get(); + // Teardown: this heap is being deleted, so it must not adopt orphans. + h.collect_for_teardown(); + let mut seg = h.segments; + h.segments = ptr::null_mut(); + while !seg.is_null() { + let next = (*seg).next; + // `next` is read BEFORE adopting and `seg` is not touched after, so + // a release during adoption is safe here. + let _ = bh.adopt_segment(seg); // re-homes queues + xheap (tid ours) + seg = next; + } + let mut hseg = h.huge_segments; + h.huge_segments = ptr::null_mut(); + while !hseg.is_null() { + let next = (*hseg).next; + let pg = &raw mut (*hseg).pages[1]; + (*pg).xheap.store(bh.delayed as usize, Ordering::Release); + (*hseg).next = bh.huge_segments; + bh.huge_segments = hseg; + hseg = next; + } + // Fold the dying heap's counters into backing so process totals hold. + merge_stats(&mut bh.stats, &h.stats); + release_heap_box(hb); + } +} + +/// `mi_heap_destroy`: drop EVERY block of the heap at once and release its +/// memory. Falls back to delete semantics when the heap was created with +/// `allow_destroy = false`. +/// +/// # Safety +/// As [`heap_delete`], and additionally no block of this heap may be used +/// afterwards (the C contract). +pub unsafe fn heap_destroy(hb: *mut HeapBox) { + // SAFETY: owner thread per contract. + unsafe { + if !(*hb).allow_destroy { + heap_delete(hb); + return; + } + debug_assert_eq!((*hb).owner_tid, thread_id()); + let h = &mut *(*hb).heap.get(); + // Spin out any in-flight FREEING remotes before the box dies; blocks + // they queued are dying with the heap, so the delayed list is simply + // discarded. + let mut seg = h.segments; + while !seg.is_null() { + let end = (*seg).next_free_slice as usize; + let mut idx = segment::HEADER_SLICES; + // Same running-pointer walk as `thread_done`'s — the sibling site, + // checked because a defect in one slice walk is a defect in every + // slice walk. Not exercised by `mleak` (this is `heap_destroy`, not + // thread teardown), so it carries no measurement of its own; it is + // the identical transformation on identical code. + let base: *mut crate::page::Page = (&raw mut (*seg).pages).cast(); + let mut slot: *mut crate::page::Page = base.add(idx); + while idx < end { + let len = ((*slot).slice_count as usize).max(1); + if (*slot).block_size > 0 { + page_set_flag(slot, XFLAG_NEVER); + (*slot).xheap.store(0, Ordering::Release); + } + idx += len; + slot = slot.wrapping_add(len); + } + let next = (*seg).next; + let _ = segment::segment_free(seg); + seg = next; + } + let mut hseg = h.huge_segments; + while !hseg.is_null() { + let pg = &raw mut (*hseg).pages[1]; + page_set_flag(pg, XFLAG_NEVER); + (*pg).xheap.store(0, Ordering::Release); + let next = (*hseg).next; + let _ = segment::huge_free(hseg); + hseg = next; + } + release_heap_box(hb); + } +} + +fn merge_stats(into: &mut crate::heap::Stats, from: &crate::heap::Stats) { + into.allocs += from.allocs; + into.frees += from.frees; + into.generic += from.generic; + into.pages_fresh += from.pages_fresh; + into.segments += from.segments; + into.huge_allocs += from.huge_allocs; + into.extends += from.extends; + into.large_allocs += from.large_allocs; + into.pages_retired += from.pages_retired; + into.segments_freed += from.segments_freed; + into.realloc_in_place += from.realloc_in_place; + into.realloc_moved += from.realloc_moved; + into.delayed_frees += from.delayed_frees; + into.reclaims += from.reclaims; +} + +/// Unregister and release a HeapBox, repairing the TLS default if needed. +/// +/// # Safety +/// `hb` live, owned by the calling thread, all contents already migrated or +/// released. +unsafe fn release_heap_box(hb: *mut HeapBox) { + heaps_unregister(hb); + if heap_tls::get() == hb { + heap_tls::set(backing_heap()); + } + // A first-class heap created before the first allocation would have been + // handed the fixed backend's static descriptor; that is not a page and + // must not be returned to the region. + if crate::ONE_REGION && prim::fixed::is_first_heap_box(hb) { + return; + } + if heap_box_cache_put(hb) { + return; + } + let size = core::mem::size_of::(); + let blockdesc = os::OsBlock { + ptr: hb.cast(), + size: os::page_align_up(size), + is_large: false, + is_zero: false, + }; + // SAFETY: hb's mapping came from create_heap's os::alloc_aligned with this + // exact size; unregistered above, no references remain. + unsafe { + let _ = os::free(blockdesc); + } +} + +// --------------------------------------------------------------------------- +// Global abandoned-segment list (spin-locked; cold path). +// --------------------------------------------------------------------------- + +/// Subprocess isolation (M6, plan §5.9): threads tagged with a subproc id +/// abandon/reclaim only within it — separate interpreters never exchange +/// segments. Id 0 is the main subprocess. +const MAX_SUBPROCS: usize = 64; + +static ABANDONED_LOCK: AtomicBool = AtomicBool::new(false); +static ABANDONED_HEADS: [AtomicPtr; MAX_SUBPROCS] = + [const { AtomicPtr::new(ptr::null_mut()) }; MAX_SUBPROCS]; +static SUBPROC_NEXT: AtomicUsize = AtomicUsize::new(1); +/// Diagnostic: segments currently abandoned (all subprocs). +pub static ABANDONED_COUNT: AtomicUsize = AtomicUsize::new(0); + +ra_thread_local! { + static SUBPROC: Cell = const { Cell::new(0) }; +} + +/// `mi_subproc_main` → 0. +pub fn subproc_main() -> usize { + 0 +} + +/// `mi_subproc_new`: allocate a fresh subprocess id. +/// +/// Exhaustion is an invalid id (`MAX_SUBPROCS`), not an assert (OH-rusty_alloc-21). +pub fn subproc_new() -> usize { + loop { + let id = SUBPROC_NEXT.load(Ordering::Acquire); + if id >= MAX_SUBPROCS { + return MAX_SUBPROCS; + } + if SUBPROC_NEXT + .compare_exchange_weak(id, id + 1, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return id; + } + } +} + +/// `mi_subproc_delete`: ids are not recycled in v1; abandoned segments of the +/// subproc are drained into the main subproc so they stay reclaimable. +pub fn subproc_delete(id: usize) { + if id == 0 || id >= MAX_SUBPROCS { + return; + } + abandoned_lock(); + // SAFETY: list surgery under the lock. + unsafe { + let mut seg = ABANDONED_HEADS[id].swap(ptr::null_mut(), Ordering::AcqRel); + while !seg.is_null() { + let next = (*seg).next; + (*seg).next = ABANDONED_HEADS[0].load(Ordering::Relaxed); + ABANDONED_HEADS[0].store(seg, Ordering::Relaxed); + seg = next; + } + } + abandoned_unlock(); +} + +/// `mi_subproc_add_current_thread`. +pub fn subproc_add_current_thread(id: usize) { + if id >= MAX_SUBPROCS { + return; + } + SUBPROC.with(|c| c.set(id)); + // Mirror onto this thread's backing heap so TEARDOWN can read the tag + // without touching the thread-local (see `HeapBox::subproc`). Tagging + // before the first allocation is the common order and is covered by + // `create_heap` reading `my_subproc()`; this covers tagging AFTER the heap + // already exists. Deliberately does NOT bootstrap a heap — a thread that + // tags itself and never allocates should not be given one. + let hb = BACKING_PTR.with(|c| c.get()); + if !hb.is_null() { + // SAFETY: hb is this thread's own live backing box; `subproc` is + // atomic, so the teardown read needs no other synchronisation. + unsafe { (*hb).subproc.store(id, Ordering::Release) }; + } +} + +fn my_subproc() -> usize { + SUBPROC.with(|c| c.get()) +} + +/// `mi_abandoned_visit_blocks`: visit live pages/blocks of a subprocess's +/// abandoned segments, holding the list lock (segments are pinned meanwhile). +pub fn abandoned_visit_blocks( + subproc_id: usize, + heap_tag: i32, + visit_blocks: bool, + f: &mut dyn FnMut(&crate::heap::AreaInfo, *mut u8, usize) -> bool, +) -> bool { + if subproc_id >= MAX_SUBPROCS { + return false; + } + abandoned_lock(); + let mut ok = true; + let mut seg = ABANDONED_HEADS[subproc_id].load(Ordering::Acquire); + while !seg.is_null() && ok { + // SAFETY: the lock pins abandoned segments (adopt also takes it). + unsafe { + ok = crate::heap::visit_segment_blocks(seg, heap_tag, visit_blocks, false, f); + seg = (*seg).next; + } + } + abandoned_unlock(); + ok +} + +fn abandoned_lock() { + while ABANDONED_LOCK + .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + core::hint::spin_loop(); + } +} + +fn abandoned_unlock() { + ABANDONED_LOCK.store(false, Ordering::Release); +} + +/// Publish `seg` to subprocess `sp`'s abandoned list. +/// +/// `sp` is passed in rather than read from `my_subproc()`: every caller is on +/// the teardown path, where the `SUBPROC` thread-local may already be gone (see +/// `HeapBox::subproc`). The owning heap box carries the tag instead. +fn abandoned_push(seg: *mut Segment, sp: usize) { + // The correctness argument behind `abandoned_pop`'s early return: if this + // ever runs on a single-context build, a thread ended, and the target's + // single-thread assertion was false all along. Worth knowing for its own + // sake -- `SingleThreadCell` and `prim::fixed` rest on the same claim. + if crate::ONE_THREAD { + unreachable!("a segment was abandoned on a build that asserted a single thread"); + } + debug_assert!(sp < MAX_SUBPROCS); + // PURGE BEFORE ORPHANING (`abandoned_page_purge`, on by default). Until + // some other thread adopts this segment it has no owner, so every page it + // touched stays resident for an unbounded time and nothing can reuse it. + // Done here, before publishing to the list, because this is the last + // instant the caller still exclusively owns the segment. + if crate::options::is_enabled(30) { + // SAFETY: still owned by the dying thread; free spans hold no live + // blocks, and each purged span is marked so reuse re-commits it. + unsafe { crate::segment::purge_free_spans(seg) }; + } + abandoned_lock(); + // SAFETY: seg is disowned; its `next` is ours to use as the list link. + unsafe { + (*seg).next = ABANDONED_HEADS[sp].load(Ordering::Relaxed); + } + ABANDONED_HEADS[sp].store(seg, Ordering::Relaxed); + ABANDONED_COUNT.fetch_add(1, Ordering::Relaxed); + abandoned_unlock(); +} + +/// Pop one abandoned segment from the CALLER's subprocess and take ownership. +/// Returns null when none are available. +pub fn abandoned_pop() -> *mut Segment { + // A single-context build has nothing to pop, by construction: a segment is + // only ever abandoned by a thread that ENDS (`thread_done`), and the + // target has asserted there is exactly one thread for the life of the + // program. Folding this to null is what makes `adopt_segment` unreachable + // from the allocation path, so the linker drops it -- 1,154 bytes on an + // ESP32-S3 that could never have run (`docs/plans/finished/firmware-code-size.md`, + // lever 1). `heap_delete` still calls it directly, and a firmware that + // deletes first-class heaps keeps it; that is a real path, not a + // cross-thread one. + if crate::ONE_THREAD { + return ptr::null_mut(); + } + let sp = my_subproc(); + if ABANDONED_HEADS[sp].load(Ordering::Acquire).is_null() { + return ptr::null_mut(); // fast no-lock exit for the common case + } + abandoned_lock(); + let seg = ABANDONED_HEADS[sp].load(Ordering::Relaxed); + if !seg.is_null() { + // SAFETY: list links are protected by the lock. + unsafe { + ABANDONED_HEADS[sp].store((*seg).next, Ordering::Relaxed); + (*seg).next = ptr::null_mut(); + (*seg).thread_id.store(thread_id(), Ordering::Release); + } + ABANDONED_COUNT.fetch_sub(1, Ordering::Relaxed); + } + abandoned_unlock(); + seg +} diff --git a/crates/rusty_alloc/src/page.rs b/crates/rusty_alloc/src/page.rs index 4ffa09d..c884659 100644 --- a/crates/rusty_alloc/src/page.rs +++ b/crates/rusty_alloc/src/page.rs @@ -1255,7 +1255,20 @@ pub unsafe fn page_extend(page: *mut Page, area: *mut u8) { // byte bound alone gives ONE block to any class above 4 KiB and one // to three to classes above 1 KiB, so a fresh medium page served one // allocation per slow-path trip. See `MIN_EXTEND`. - let take = ((reserved >> span_shift).max(MIN_EXTEND)).min(reserved - capacity); + // + // And at least what the page already holds (`capacity`): the batch + // GROWS geometrically, 4 KiB, 4 KiB, 8, 16 ... Each extension costs a + // slow-path trip (the generic path, `grow_front`'s setup: ~117 Ir + // before the first block), and the linking itself is only 2.75 Ir a + // block (LLVM unrolls it x4), so the trips were the cost. Measured, + // allocator self Ir: Endless Sky -4.50 %, lua -1.94 %, perl -1.62 %, + // python -6.4 %; thread life flat (a short thread still carves one + // 4 KiB page); peak RSS within +-0.8 %. Carved-but-unused blocks stay + // bounded by the blocks already carved. A FIXED 8 KiB batch got less + // (ES -2.92 %) and cost thread life +10 % (16,923 -> 18,643 Ir): every + // short thread carved twice what it used. + let take = + ((reserved >> span_shift).max(MIN_EXTEND).max(capacity)).min(reserved - capacity); let start = area.add(capacity * bsize); // Link the fresh blocks in address order. // diff --git a/crates/rusty_alloc_api/CHANGELOG.md b/crates/rusty_alloc_api/CHANGELOG.md index 339006c..ce93a75 100644 --- a/crates/rusty_alloc_api/CHANGELOG.md +++ b/crates/rusty_alloc_api/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Performance + +- `GlobalAlloc::alloc` moves its natural-alignment arm out of line, so the + inlined method carries one malloc fast path, not two (maps -0.33 %, + overaligned -1.81 %); over-aligned `realloc` inlines its usable-size + query (-2.06 %); `dealloc` uses `free_inline_flags_first`. The core's + changelog has the rest. + ## [2.2.5](https://github.com/Remade-With-Rust/rusty_alloc/compare/rusty_alloc-api-v2.2.4...rusty_alloc-api-v2.2.5) - 2026-10-08 ### Fixed diff --git a/crates/rusty_alloc_api/src/lib.rs b/crates/rusty_alloc_api/src/lib.rs index 9633b10..599bf57 100644 --- a/crates/rusty_alloc_api/src/lib.rs +++ b/crates/rusty_alloc_api/src/lib.rs @@ -169,20 +169,20 @@ unsafe impl GlobalAlloc for RustyAlloc { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { if layout.align() <= WORD { rusty_alloc::alloc::malloc(layout.size()) - } else if layout.align() <= NATURAL_ALIGN { - // See `NATURAL_ALIGN`: a class of two words or more is already - // aligned this far, so only a one-word request needs raising. - rusty_alloc::alloc::malloc(layout.size().max(NATURAL_ALIGN)) - } else { + } else if layout.align() > NATURAL_ALIGN { // SAFETY: `Layout` guarantees a power-of-two alignment, the one // precondition `malloc_aligned_pow2` adds over `malloc_aligned`. unsafe { rusty_alloc::alloc::malloc_aligned_pow2(layout.size(), layout.align()) } + } else { + alloc_natural(layout.size()) } } #[inline] unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { - // `free_inline`, not `free`: `dealloc` IS a free and does nothing + // `free_inline_flags_first` (the body of `free_inline`, in the test + // order that keeps `__rust_dealloc` inlinable; see its doc), not + // `free`: `dealloc` IS a free and does nothing // else, the case `free_inline` exists for (the LD_PRELOAD export is // the other). Through `free` every Rust deallocation paid a `jmp` // into it and its null test; `GlobalAlloc` never passes null, and the @@ -191,7 +191,7 @@ unsafe impl GlobalAlloc for RustyAlloc { // and is freed once. unsafe { core::hint::assert_unchecked(!ptr.is_null()); - rusty_alloc::alloc::free_inline(ptr) + rusty_alloc::alloc::free_inline_flags_first(ptr) } } @@ -255,3 +255,14 @@ unsafe impl GlobalAlloc for RustyAlloc { } } } + +/// `GlobalAlloc::alloc` for an alignment above one word up to +/// `NATURAL_ALIGN`, out of line. Inline, this arm was a SECOND copy of the +/// malloc fast path in every `__rust_alloc`, which kept the method too big for +/// LLVM to inline into its callers. See `NATURAL_ALIGN`: a class of two words +/// or more is already aligned this far, so only a one-word request needs +/// raising. +#[inline(never)] +fn alloc_natural(size: usize) -> *mut u8 { + rusty_alloc::alloc::malloc(size.max(NATURAL_ALIGN)) +} diff --git a/crates/rusty_alloc_ffi/src/lib.rs b/crates/rusty_alloc_ffi/src/lib.rs index c18e49e..795c6fa 100644 --- a/crates/rusty_alloc_ffi/src/lib.rs +++ b/crates/rusty_alloc_ffi/src/lib.rs @@ -2050,7 +2050,17 @@ pub fn new_impl(size: usize) -> *mut c_void { // would keep `size` alive across the slow-path call, and that single // live value gives every `operator new` export a frame. See // `alloc::malloc_or`. - alloc::malloc_or(size, new_oom).cast() + alloc::malloc_or_with::(size).cast() +} + +/// [`new_oom`] as a compile-time handler, so the cold arm calls it directly. +struct NewOom; + +impl alloc::OnOom for NewOom { + #[inline(always)] + fn on_oom(size: usize) -> *mut u8 { + new_oom(size) + } } /// `operator new` ran out of memory: collect once, retry, and abort if that diff --git a/docs/opps.md b/docs/opps.md index da6411c..bbcc3dc 100644 --- a/docs/opps.md +++ b/docs/opps.md @@ -518,6 +518,86 @@ a `let _ = retire_span(..)` missing its same-line terminal comment (semgrep `discarded-lifecycle-result`, CI-blocking); `corpus/linux-gates.sh` hard-coded `/mnt/c/...` and exited 1 before running anything. +## Vein census 3, 2026-10-08 (real programs, by function) + +**Method.** Callgrind on real programs, not synthetic ops: perl, sqlite, lua, +python and the Endless Sky start-up through the override (`LD_PRELOAD` on +valgrind itself: through `env`, valgrind traces `env` and not the exec), plus +the four Rust `GlobalAlloc` workloads. Per function: self Ir, call count, +self/call and caller->callee edges (`F:/ra-corpus-tmp/veins/cgcensus.py`, +`census3.sh`). Build: `main` at 2.2.5. mimalloc 2.4.5 through the same +parser: rusty_alloc is cheaper PER CALL at every comparable site (free 21.0 vs +25.0, malloc 14.9 vs 15.9, the page carve 1.98 M vs 3.14 M total on perl, the +lua realloc path 11.2 M vs 27.7 M, the Endless Sky cross-thread free +5.5 M vs ~15.3 M). So these veins are absolute costs, not gaps to the +oracle. + +Allocator share of the whole program: Endless Sky 57.4 M Ir (5.62 %), perl +24.7 M (3.19 %), lua 23.1 M (3.85 %), sqlite 4.6 M (1.45 %), python 0.17 M. +The two fast paths are most of it (ES: delete 19.8 and new 14.8 Ir/call, 41 % +and 31 % of the allocator) and four campaigns have mined them; they are not +on this list except where a census line names a specific cost. + +| # | vein | evidence (census) | ceiling | first probe | +|---|---|---|---|---| +| 1 | **Carving builds a whole free list up front.** `page_extend` links every block of the extension before one is used | `grow_front` perl 1.98 M (8.0 % of allocator; 1.84 M in page.rs), ES 1.91 M, lua 1.20 M; the thread-start walk spends most of its 911 Ir/call in the same loop | most of the carve: ~1.8 M on perl, ~1.9 M on ES | a bump-pointer carve (allocate from the uncarved tail, link nothing until freed). A throwaway that only times the loop sizes it first | +| 2 | **Endless Sky: 114 k LOCAL frees take the slow path.** `free_general` 175,393 calls, only 61,643 of them `remote_free` | ES `free_general` 4.37 M self (24.9/call, 7.6 % of allocator) | ~2.8 M if the common reason stays on the fast path | a per-reason counter: which `SLOW_FREE` bit (`IN_FULL` likely, or `SINGLE_BLOCK`) | +| 3 | **Rust: `GlobalAlloc::alloc` is not inlined into `__rust_alloc`; `dealloc` is.** | maps: `__rust_alloc` 4.0 Ir/call self, then a call to `RustyAlloc::alloc` (18.5 self) on 16,026 calls; hashbrown calls it out of line too | ~4-5 Ir per Rust allocation, in every Rust consumer | `#[inline(always)]` on `GlobalAlloc::alloc` (or split body/symbol); whole-program Ir + `.text` on all four workloads | +| 4 | **The generic slow path's 50-63 Ir of self cost per trip.** | ES 70,187 trips x 63.1 = 4.43 M (7.7 %): heap.rs 2.25 M, page.rs 0.80 M, **bins.rs 0.75 M (10.7/trip: the bin recomputed from the size)**; perl 574 k, lua 405 k | the bin recompute, ~0.75 M on ES; more after a line census | pass the bin the fast path already computed; per-line census of heap.rs | +| 5 | **C++ `operator new`'s miss takes an extra hop.** new -> `malloc_or_slow` (12 Ir) -> `malloc_slow` (7) -> generic, where `malloc` goes straight to `malloc_slow` | ES `malloc_or_slow` 70,115 x 12.0 = 841 k (1.5 %) | ~0.8 M on ES; every C++ program under the override | route the miss straight to `malloc_slow`, with the null/throw test after | +| 6 | **`realloc_live` self cost, re-measured.** "At its floor" was decided in 2026-08 against another shape | lua 60,051 calls x 55 = 3.30 M (14.3 % of lua's allocator): alloc.rs 29/call, page.rs 9, **segment.rs 5 (a segment re-resolve?)**, init.rs 2 (TLS heap) | the re-resolve plus whatever a line census finds | per-line census; check whether the segment and heap are already in hand from `realloc` | +| 7 | **Thread start/exit fixed cost and one OS allocation per thread.** | rust-threads, 200 threads: walk 911 x 2, `collect_inner` 436 x 2, `create_heap_uninstalled` 163, `segment_free` 105, `thread_done_one` 94, `reseed` 67, and `os::alloc_aligned`/`prim::alloc` 203 calls (an OS call per thread) | ~3.9 k Ir and a syscall pair per thread; thread pools pay it per spawn | find what the per-thread OS allocation is, and reuse it from exited threads | +| 8 | **Over-aligned Rust realloc asks `usable_size` out of line, then re-allocates.** | overaligned: `__rust_realloc` -> `usable_size` 5,125 x 13.0 = 66.6 k (27 % of that workload's allocator Ir), then `alloc` 2,678 x 60 | most of the 66.6 k, plus the moves that fit in place | an aligned in-place realloc that resolves the page once | +| 9 | **Win back 2.2.5's +3 Ir/free on MSVC builds that unwind.** The goto is banned there, but an `asm!` with an OUTPUT operand is not | 2.2.5: plain-Rust decrement = +3 Ir per local free (small 52.26 -> 55.26 forced on Linux) | up to 3 Ir/free on affected builds (rusty_sloth) | `sub dword ptr [..], 1` + `setle` into an output, branch in Rust; forced on Linux, then the rusty_sloth corpus row | +| 10 | **A fat heap's thread-exit collect walks every page.** | ES: ONE `thread_done_one` -> `collect_inner` = 1.82 M Ir (page.rs 1.62 M); mimalloc pays the same (`_mi_page_free_collect` 1.64 M) | ~1.8 M per exit of a big heap | abandon whole segments on a dying heap without per-page collection, if adoption can collect lazily | + +**Rules for working these** (from the skill and this campaign): paired, +same-session measurements only; quote at least two shapes per probe (a real +program AND opscan); the work-parity anchors are the program's output and the +call counts above; record every refutation with its number. + +### Vein census 3 results, 2026-10-08 + +Final tree against 2.2.5, same session, allocator self Ir (`real.sh`; outputs +identical): + +| program | 2.2.5 | after | delta | +|---|---:|---:|---:| +| Endless Sky start-up | 57,380,635 | 53,453,604 | **-6.85 %** | +| lua | 23,149,581 | 22,632,842 | **-2.23 %** | +| perl | 24,668,474 | 24,262,947 | **-1.64 %** | +| sqlite | 4,592,621 | 4,587,503 | -0.11 % | +| python | 168,937 | 155,977 | -7.7 % | + +Rust `GlobalAlloc` workloads, whole program (checksums equal): overaligned +-4.24 %, threads -1.53 %, maps -0.36 %, trees -0.01 %. Opscan: realloc +231.89 -> 227.97, xthread 77.09 -> 74.82, aligned 83.75 -> 83.00, calloc +93.75 -> 93.00, mixed 94.26 -> 93.56, small/big unchanged; **2 MiB op 142.06 -> +144.06 and 64 MiB pair 834 -> 836 (+2: vein 2's trade, below)**. Peak RSS +within +-0.4 % (perl, lua, ES); 28-thread battle replay 1,752.6 -> 1,744.1 MB. + +| vein | result | what landed | +|---|---|---| +| 1 carve | **LANDED**, the biggest | geometric extension batches (`page_extend`: at least what the page holds). ES -4.50 %, lua -1.94 %, perl -1.62 %. The link loop was already 2.75 Ir/block (unrolled x4); the slow-path TRIPS were the cost. A fixed 8 KiB batch: ES -2.92 % but thread life +10 % (refuted) | +| 2 slow frees | **LANDED** | the census premise was wrong: the 175 k were CROSS-thread frees into FULL pages (`remote_free` inlined in `free_general`). C exports test the owner first (`free_inline`); Rust keeps the 2.2.5 order (`free_inline_flags_first`), because the bigger remote arm stopped `__rust_dealloc` inlining into drop glue (`trees` +2.88 %). ES -515,705, xthread -2.9 %. Trade: a LOCAL free into a slow page (large span, huge, aligned, full) pays the owner compare first, +2 Ir (the 2 MiB and 64 MiB ops). Refuted: unalign inlined into the remote arm (+3 Ir on EVERY free); one cold call for all remote frees (-43,649 only) | +| 3 Rust alloc inlining | **LANDED** | only the natural-alignment arm out of line (it was a second inlined copy of malloc). maps -0.33 %, overaligned -1.81 %. Refuted: `#[inline(always)]` (maps +0.98 %), all non-word arms out of line (overaligned +0.93 %), one merged malloc call (maps +1.33 %) | +| 4 bin recompute | **LANDED** | `SMALL_BIN` table by word count, indexed with the word count the generic path already computes. ES -0.70 %, then -73 k more with the shared word count | +| 5 operator new hop | **LANDED** | `malloc_or_with::`: the cold arm's frame 12 -> 7 instructions (a typed handler, not a `fn` pointer). ES -419,182 | +| 6 realloc_live | **LANDED** (small) | the page resolved for the usable size is handed to the free. lua -60,025, opscan realloc -2.00 | +| 7 thread start/exit | **LANDED** | an 8-slot heap-box cache (upstream `TD_CACHE_SIZE`). Thread life -1.1 %, Rust threads -1.46 %, minor faults ~137 -> ~117 per run | +| 8 over-aligned realloc | **LANDED** | `#[inline]` on `usable_size`. overaligned -2.06 % (isolated) | +| 9 MSVC unwind | **LANDED** | `sub` + `setle` into an output on MSVC with `panic = "unwind"`: +2 over the goto where 2.2.5's plain decrement was +3 (forced on Linux). rusty_sloth builds (registry-identity corpus row), MSVC unwind IR still has 0 `callbr` | +| 10 thread-exit collect | **REFUTED, inherent** | the cost is the cross-thread drain (each remote-freed block walked once, at its collect; teardown is where the backlog is). mimalloc pays the same 1.64 M. The per-step `n > used` check also bounds a corrupted (cyclic) chain, so it stays | + +Gates: workspace `--all-features` 177/177, linux-gates 42/0, embedded 15/15, +clippy (Windows + Linux targets, all features), fmt, semgrep, wasm self-test +and size (21,175 B gz, inside the +3 % ratchet), gate self-test 11/11, miri +gate + `openheimer` 37/37 under miri, mimalloc-bench sweep 19/19, churn 5/5, +real-world programs byte-identical across ra/mi/sys, alloc-eval checks on all +8 traces, rusty_sloth (crates.io identity, `--release --features cuda`). +One unexplained failure in the first of 27 Windows test runs (log lost), not +reproduced in the 26 after it. + ## Banked ### 8. Collect-loop double `block_next` — LANDED 2026-08-20 diff --git a/tools/unsafe-baseline.txt b/tools/unsafe-baseline.txt index 02a3289..f4df05e 100644 --- a/tools/unsafe-baseline.txt +++ b/tools/unsafe-baseline.txt @@ -1,4 +1,4 @@ - 110 crates/rusty_alloc/src/alloc.rs + 123 crates/rusty_alloc/src/alloc.rs 15 crates/rusty_alloc/src/arena.rs 80 crates/rusty_alloc/src/heap.rs 42 crates/rusty_alloc/src/init.rs From bd2391cf5f3f1765725c46314c1fd5fb7563b3ce Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 7 Oct 2026 23:18:43 -0700 Subject: [PATCH 2/2] style(init): keep init.rs's CRLF line endings cargo fmt rewrote init.rs (the repo's one CRLF file) to LF in the previous commit, which buried its 52-line change in a 2,547-line diff. Restored; cargo fmt --check accepts either. Co-Authored-By: Claude Opus 5.5 --- crates/rusty_alloc/src/init.rs | 2594 ++++++++++++++++---------------- 1 file changed, 1297 insertions(+), 1297 deletions(-) diff --git a/crates/rusty_alloc/src/init.rs b/crates/rusty_alloc/src/init.rs index 4579bbf..e28b29e 100644 --- a/crates/rusty_alloc/src/init.rs +++ b/crates/rusty_alloc/src/init.rs @@ -1,1297 +1,1297 @@ -//! Thread + heap lifecycle (mirrors upstream `init.c` + the abandonment side -//! of `arena-abandon.c`). -//! -//! Every thread gets a lazily-created [`HeapBox`] (one OS page from the prim -//! layer — never from the allocator itself, so bootstrap can't recurse). The -//! fast path reaches it through a const-init `thread_local!` `Cell` pointer, -//! which the R1 spike measured at atomic-load parity (0.3 ns) — no dtor is -//! registered on that key (Cell is !Drop), so accessing it never allocates. -//! -//! Thread EXIT runs through the prim TLS destructor (`FlsAlloc` / -//! `pthread_key_create`, built in M1 for exactly this): drain delayed frees, -//! retire what died, transition every surviving page to `NEVER` (spinning out -//! in-flight `FREEING` remotes — the loom-modeled teardown order), zero the -//! segments' thread id, and publish them on the global abandoned list. Any -//! other thread's allocation slow path adopts them later. - -use core::cell::{Cell, UnsafeCell}; -use core::ffi::c_void; -use core::ptr; -use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; - -use crate::heap::Heap; -use crate::page::{ - DelayedList, XFLAG_NEVER, page_collect, page_collect_and_set_flag, page_set_flag, -}; -use crate::segment::{self, Segment}; -use crate::{os, prim}; - -/// A heap plus the cross-thread fields remote threads may touch. Remote -/// threads only ever dereference `delayed` (via page `xheap` addresses) — -/// never the heap itself — which is what makes `&mut` on the inner heap sound. -#[repr(C)] -pub struct HeapBox { - /// Cross-thread delayed-free list (remote-reachable). MUST stay the first - /// field: pages store `&delayed` in `xheap`, and owner routing recovers - /// the box by container-of (delayed at offset 0 ⇒ the addresses coincide). - pub delayed: DelayedList, - /// Next box in the global registry (guarded by the registry lock). - /// AtomicPtr, not usize: reachability (miri's leak check and honesty - /// about what is cached vs leaked) follows pointers, not integers. - pub next_box: AtomicPtr, - /// Owning thread id (heaps allocate only from their creating thread). - pub owner_tid: usize, - /// The subprocess this heap's segments are abandoned into. - /// - /// Mirrored from the `SUBPROC` thread-local so that TEARDOWN never has to - /// read it. `thread_done` runs from a platform TLS destructor - /// (`pthread_key_create` / `FlsAlloc`), and the destruction order between - /// *that* callback and Rust's own `thread_local!` storage is unspecified on - /// every platform. On aarch64-apple-darwin it is observably wrong: - /// `my_subproc()` reads back 0 inside `thread_done`, so a tagged thread's - /// segments landed in the MAIN subprocess and `mi_subproc_*` isolation was - /// silently lost — the abandon fired correctly, only its destination was - /// wrong. The heap box is our own mapping and is alive for the whole of - /// teardown (it is the value passed *to* the destructor), so reading the tag - /// from here is correct on every target instead of accidentally correct on - /// one. - pub subproc: AtomicUsize, - /// Heap tag (`mi_heap_new_ex`; reported by the block visitor). - pub tag: i32, - /// Whether `mi_heap_destroy` may drop live blocks (else it delegates to - /// delete semantics). - pub allow_destroy: bool, - /// The owner-only heap state. - pub heap: UnsafeCell, -} - -const _: () = assert!(core::mem::offset_of!(HeapBox, delayed) == 0); - -/// Wrapper so the empty sentinel box can be a `static`. -#[repr(transparent)] -pub struct EmptyHeapBox(HeapBox); - -// SAFETY: the sentinel is never written and never escapes to remote threads: -// no page ever stores its `delayed` address in `xheap` (pages are re-homed -// only by REAL heaps), its `owner_tid` of 0 matches no live thread, and every -// reader goes through raw-pointer READS — `heap_box_fast` documents that no -// `&mut` may ever be formed to it. It exists purely so the malloc fast path -// has a valid heap to read before this thread has created one. -unsafe impl Sync for EmptyHeapBox {} - -/// The "no heap yet" sentinel: every `direct` slot holds the empty PAGE -/// sentinel, so a fresh thread's first malloc falls through to the generic -/// path — which detects this box and performs the real thread init — without -/// the fast path ever testing for it. This is upstream's `_mi_heap_empty` -/// trick, and the exact design our own [`crate::page::EMPTY_PAGE`] already -/// uses one level down: replace a null-test on the hot path with a sentinel -/// whose data routes the miss to the slow path for free. -/// -/// `export_name` (not mangled) because the x86-64 Linux TLS slot's `.tdata` -/// initializer below names it in assembly; `.hidden` there keeps it out of -/// the cdylib's export table. -/// -/// **In flash on a one-region target.** The sentinel is read and never -/// written — `heap_box_fast` documents that contract, the fast path only -/// writes after a non-null pop and every `direct` slot here points at the -/// empty page whose free list is null, and no page ever carries its address -/// in `xheap`. Its type has interior mutability (a `DelayedList`, an -/// `AtomicPtr`, the `UnsafeCell`), which is why rustc places it in -/// `.data` by default: 1,752 bytes paid once in flash for the image and once -/// more in RAM for the loader's copy, the largest single item left on the -/// ESP32-S3 after 2.0.2 and RAM the linker takes from the stack. Placing it -/// in `.rodata.*` keeps the flash copy and drops the RAM one. Any write -/// would fault loudly against the flash cache, which is the right way for -/// the contract to fail. Bare metal only — a hosted target may need to -/// relocate the pointers inside it at load, and has RAM to spare. -#[cfg_attr( - all( - ra_single_threaded, - not(miri), - not(windows), - not(unix), - not(target_arch = "wasm32") - ), - unsafe(link_section = ".rodata.rusty_alloc_empty_heap_box") -)] -#[unsafe(export_name = "__ra_empty_heap_box")] -pub static EMPTY_HEAP_BOX: EmptyHeapBox = EmptyHeapBox(HeapBox { - delayed: DelayedList::new(), - next_box: AtomicPtr::new(ptr::null_mut()), - owner_tid: 0, - subproc: AtomicUsize::new(0), - tag: 0, - allow_destroy: false, - heap: UnsafeCell::new(Heap::new()), -}); - -/// Pointer to the shared empty heap box. -#[inline] -pub const fn empty_heap_box_ptr() -> *mut HeapBox { - (&raw const EMPTY_HEAP_BOX.0).cast_mut() -} - -/// Recover the owning HeapBox from a page's `xheap` value (the container-of -/// trick over the offset-0 delayed list). -/// -/// # Safety -/// `xheap` must be a live HeapBox's delayed-list address (nonzero). -#[inline] -pub unsafe fn box_of_xheap(xheap: usize) -> *mut HeapBox { - debug_assert!(xheap != 0); - xheap as *mut HeapBox -} - -// Global registry of live heap boxes: keeps every heap (and through it every -// segment) reachable-from-static — the process-wide stats walk of M7, and -// what makes miri's reachability-based leak check reflect reality (an -// allocator's arenas at process exit are cached state, not leaks). -static HEAPS_LOCK: AtomicBool = AtomicBool::new(false); -static HEAPS_HEAD: AtomicPtr = AtomicPtr::new(ptr::null_mut()); - -fn heaps_lock() { - while HEAPS_LOCK - .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) - .is_err() - { - core::hint::spin_loop(); - } -} - -fn heaps_unlock() { - HEAPS_LOCK.store(false, Ordering::Release); -} - -fn heaps_register(hb: *mut HeapBox) { - heaps_lock(); - // SAFETY: hb is fully initialized and not yet published. - unsafe { - (*hb) - .next_box - .store(HEAPS_HEAD.load(Ordering::Relaxed), Ordering::Relaxed); - } - HEAPS_HEAD.store(hb, Ordering::Relaxed); - heaps_unlock(); -} - -fn heaps_unregister(hb: *mut HeapBox) { - heaps_lock(); - let mut cur = HEAPS_HEAD.load(Ordering::Relaxed); - if cur == hb { - // SAFETY: links guarded by the lock. - unsafe { - HEAPS_HEAD.store((*hb).next_box.load(Ordering::Relaxed), Ordering::Relaxed); - } - } else { - while !cur.is_null() { - // SAFETY: links guarded by the lock; nodes are live boxes. - unsafe { - let nxt = (*cur).next_box.load(Ordering::Relaxed); - if nxt == hb { - (*cur) - .next_box - .store((*nxt).next_box.load(Ordering::Relaxed), Ordering::Relaxed); - break; - } - cur = nxt; - } - } - } - heaps_unlock(); -} - -/// Unlink and return one live heap box owned by `tid`, or null. -/// -/// The lock is released before return so the caller can abandon the heap -/// without re-entering `HEAPS_LOCK` (abandon walks pages and can allocate). -fn take_heap_owned_by(tid: usize) -> *mut HeapBox { - heaps_lock(); - let mut prev: *mut HeapBox = ptr::null_mut(); - let mut cur = HEAPS_HEAD.load(Ordering::Relaxed); - while !cur.is_null() { - // SAFETY: links and boxes are live under the lock. - unsafe { - let nxt = (*cur).next_box.load(Ordering::Relaxed); - if (*cur).owner_tid == tid { - if prev.is_null() { - HEAPS_HEAD.store(nxt, Ordering::Relaxed); - } else { - (*prev).next_box.store(nxt, Ordering::Relaxed); - } - heaps_unlock(); - return cur; - } - prev = cur; - cur = nxt; - } - } - heaps_unlock(); - ptr::null_mut() -} - -/// Storage for the calling thread's default-heap pointer. -/// -/// On x86-64 Linux this is a RAW ELF TLS symbol reached with the -/// **initial-exec** model rather than `thread_local!`'s general-dynamic. That -/// was the last `__tls_get_addr` call on the malloc fast path — 7.3 M -/// instructions on perl, a third of everything still separating us from -/// mimalloc after the M10b free-path bricks. Stable Rust has no -/// `-Z tls-model`, so the symbol is declared in `global_asm!` and the linker -/// resolves its offset. -/// -/// **Why this is sound where a thread-pointer-keyed side table is not.** The -/// storage IS the thread's own TLS block. Every thread gets a fresh block -/// initialised from the `.tdata` template (the empty-heap sentinel) when it -/// is created, so a recycled TCB cannot hand a new thread a dead thread's -/// heap — the staleness -/// question that made the keyed-cache design a P0 risk simply does not arise -/// here. What initial-exec costs instead is a LOAD-TIME constraint: it needs a -/// slot in the static TLS block, so a `dlopen` long after startup could fail -/// to load us. That fails loudly at load time rather than corrupting memory, -/// and it is the same trade upstream mimalloc ships -/// (`__attribute__((tls_model("initial-exec")))` on `_mi_heap_default`). -#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] -mod heap_tls { - use super::HeapBox; - - // An 8-byte thread-local slot. `.tdata` (not `.tbss`): the slot is - // INITIALISED to the empty-heap sentinel, so every new thread reads a - // valid heap pointer from its very first malloc and the fast path needs - // no "not yet created" test at all — the sentinel's empty direct table - // routes the first allocation to the generic path, which performs the - // real init there (upstream's `_mi_heap_empty` design). The loader - // relocates the `.quad` in the TLS template once at load time; every - // thread's block is then copied from that template. `.hidden` makes both - // symbols non-preemptible, which is what permits the initial-exec - // (GOTTPOFF) relocation instead of a general-dynamic call, and keeps the - // sentinel out of the cdylib's export table. - core::arch::global_asm!( - ".section .tdata,\"awT\",@progbits", - ".globl __ra_tls_heap", - ".hidden __ra_tls_heap", - ".hidden __ra_empty_heap_box", - ".type __ra_tls_heap,@object", - ".p2align 3", - "__ra_tls_heap:", - ".quad __ra_empty_heap_box", - ".size __ra_tls_heap,8", - ".text", - ); - - /// Address of THIS thread's slot: the thread pointer plus the offset the - /// dynamic linker wrote into the GOT. - #[inline(always)] - fn slot() -> *mut *mut HeapBox { - let off: usize; - // SAFETY: reads the GOT entry the dynamic linker relocated at load - // time, before any of our code can run, and never writes. It is - // constant for the life of the process, hence `pure` + `readonly` — - // that is what lets LLVM hoist and CSE the address computation - // instead of treating the block as an optimisation barrier. - unsafe { - core::arch::asm!( - "mov {o}, qword ptr [rip + __ra_tls_heap@GOTTPOFF]", - o = out(reg) off, - options(nostack, preserves_flags, readonly, pure), - ); - } - // `thread_id()` IS the fs base on this target. TLS offsets are - // negative there (the block grows down from the thread pointer), so - // this adds a two's-complement offset and must wrap. - core::ptr::with_exposed_provenance_mut(super::thread_id().wrapping_add(off)) - } - - /// Read this thread's heap pointer in TWO instructions. - /// - /// The obvious form — `slot().read()` — costs FOUR: load the offset from - /// the GOT, read the thread pointer from `fs:0`, add them, dereference. - /// But x86 performs that addition IN THE ADDRESSING MODE: `fs:[reg]` is a - /// segment-relative load, so the explicit `fs:0` read and the add both - /// disappear. Measured at 4.00 Ir per malloc before this change — a - /// quarter of our entire malloc-side deficit spent just locating the heap. - #[inline(always)] - pub fn get() -> *mut HeapBox { - let v: *mut HeapBox; - // SAFETY: reads the GOT entry the loader relocated at startup, then - // loads 8 aligned bytes of this thread's own TLS block through the fs - // segment. Both are reads; nothing is written. - unsafe { - core::arch::asm!( - "mov {t}, qword ptr [rip + __ra_tls_heap@GOTTPOFF]", - "mov {o}, qword ptr fs:[{t}]", - t = out(reg) _, - o = out(reg) v, - options(nostack, preserves_flags, readonly), - ); - } - v - } - - #[inline(always)] - pub fn set(hb: *mut HeapBox) { - // SAFETY: as `get` — the slot is owned exclusively by this thread. - unsafe { slot().write(hb) } - } -} - -/// Every other target keeps the `thread_local!`: Windows TLS is an index into -/// a per-thread array (no `__tls_get_addr` to remove), and miri models the -/// macro but not a hand-rolled TLS symbol. -#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))] -mod heap_tls { - use super::HeapBox; - use core::cell::Cell; - - ra_thread_local! { - /// Fast-path heap pointer. Const-init + !Drop ⇒ plain TLS access, no - /// lazy-init branch, no allocation ever. Initialised to the - /// empty-heap SENTINEL (never null) so the malloc fast path can read - /// through it unconditionally; see `EMPTY_HEAP_BOX`. - static HEAP_PTR: Cell<*mut HeapBox> = const { Cell::new(super::empty_heap_box_ptr()) }; - } - - #[inline(always)] - pub fn get() -> *mut HeapBox { - HEAP_PTR.with(|c| c.get()) - } - - #[inline(always)] - pub fn set(hb: *mut HeapBox) { - HEAP_PTR.with(|c| c.set(hb)); - } -} - -ra_thread_local! { - /// Cached OS thread id. `free` needs the calling thread's id on EVERY - /// call to route local-vs-remote; the raw `prim::thread_id()` is a libc - /// call (`pthread_self` through the PLT from a cdylib) measured at - /// 1.41 ns vs 0.25 ns for this cache — ~18-20% of a whole malloc+free - /// pair (M9 probe, both platforms). 0 = not yet cached; no real thread id - /// is 0 on either platform. - static TID: Cell = const { Cell::new(0) }; -} - -/// The calling thread's id — read STRAIGHT FROM THE THREAD-POINTER REGISTER -/// where the target allows it. -/// -/// Why this and not `thread_local!`: the shipping artifact is a **cdylib** -/// (LD_PRELOAD). Rust's `thread_local!` in a shared library compiles to the -/// general-dynamic TLS model, so every access is a CALL into `ld.so`'s -/// `__tls_get_addr` — the per-function instruction profile put that call at -/// 12.97 M Ir, 1.96% of a whole lua run, comparable to our entire `free`. -/// (The earlier ns-level probe missed it because it measured TLS inside an -/// EXECUTABLE, where the model is local-exec: a register offset, no call. -/// Measure the artifact you ship.) -/// -/// A ceiling probe with `-Z tls-model=initial-exec` valued removing those -/// calls at 2.00% of our instructions and 33% of the gap to mimalloc; this -/// gets the same effect on stable, for the id, with no linker involvement — -/// the thread pointer is one register read. It is exactly what mimalloc's -/// `_mi_thread_id()` does. -/// -/// Uniqueness: the value is the thread-control-block address, unique among -/// LIVE threads. It can be recycled after a thread dies, which is sound here -/// for the same reason it is in mimalloc: a dying thread abandons its -/// segments (`thread_id` stored as 0) before its TCB can be reused, so a -/// recycled id can never match a stale segment. -#[inline(always)] -pub fn thread_id() -> usize { - #[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] - { - let tp: usize; - // SAFETY: reads the TCB self-pointer at fs:0, which the x86-64 Linux - // ABI guarantees is present and thread-unique. No memory is written. - unsafe { - core::arch::asm!("mov {}, fs:0", out(reg) tp, - options(nostack, preserves_flags, readonly)); - } - tp - } - #[cfg(all(target_arch = "x86_64", target_os = "windows", not(miri)))] - { - let teb: usize; - // SAFETY: gs:0x30 is the TEB self-pointer on x86-64 Windows — - // present in every thread, unique, read-only here. - unsafe { - core::arch::asm!("mov {}, gs:0x30", out(reg) teb, - options(nostack, preserves_flags, readonly)); - } - teb - } - // AArch64 splits by OS: the register holding the thread pointer is NOT the - // same one on Darwin as everywhere else. - #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple"), not(miri)))] - { - let tp: usize; - // SAFETY: on the standard AArch64 ABI (Linux, Android, BSD) tpidr_el0 - // IS the thread pointer; read-only. - unsafe { - core::arch::asm!("mrs {}, tpidr_el0", out(reg) tp, - options(nostack, preserves_flags, readonly)); - } - tp - } - #[cfg(all(target_arch = "aarch64", target_vendor = "apple", not(miri)))] - { - let tp: usize; - // SAFETY: read-only system-register read. - // - // DARWIN IS NOT THE STANDARD AArch64 ABI. Apple puts the thread pointer - // in tpidrRO_el0 and uses tpidr_el0 for the CPU/cluster id. Reading - // tpidr_el0 here — as every other AArch64 target correctly does — was a - // memory-safety bug, measured on macOS 26 / M-series: - // - // * tpidr_el0 returns small values (0x1002, 0x2005, …), not pointers; - // * it CHANGES under a single thread as that thread migrates between - // cores (5 distinct values over 3M reads on the main thread); - // * and 8 live threads produced only 5 distinct values — DIFFERENT - // THREADS COLLIDED. - // - // thread_id() is the allocator's ownership identity: `segment.thread_id` - // decides whether a free takes the local (unsynchronised) path or the - // remote one. Colliding ids let one thread free into another thread's - // segment through the owner path, racing non-atomic page state — heap - // corruption, not merely a lost optimisation. A drifting id separately - // makes a thread stop recognising its own segments. - // - // tpidrro_el0 is the real thread pointer: stable within a thread and - // unique across live threads (it is `pthread_self() + 0xe0`). Apple - // documents the LOW 3 BITS as the current CPU number, so they must be - // masked off — that is precisely the drift observed above. - unsafe { - core::arch::asm!("mrs {}, tpidrro_el0", out(reg) tp, - options(nostack, preserves_flags, readonly)); - } - tp & !0b111 - } - // Every other target: the cached-TLS path (still far cheaper than asking - // the OS on each call). - #[cfg(not(any( - all(target_arch = "x86_64", target_os = "linux", not(miri)), - all(target_arch = "x86_64", target_os = "windows", not(miri)), - all(target_arch = "aarch64", not(miri)) - )))] - { - let t = TID.with(|c| c.get()); - if t != 0 { t } else { init_tid() } - } -} - -#[cold] -#[cfg_attr( - any( - all(target_arch = "x86_64", target_os = "linux", not(miri)), - all(target_arch = "x86_64", target_os = "windows", not(miri)), - all(target_arch = "aarch64", not(miri)) - ), - allow(dead_code) -)] -fn init_tid() -> usize { - let t = prim::thread_id(); - TID.with(|c| c.set(t)); - t -} - -/// Snapshot-visit every registered heap (stats aggregation). The visitor -/// receives a COPY of the heap's counters. -/// -/// **This read is racy by design and `volatile` does not make it less so** — -/// clippy's `volatile_composites` is right to flag it, and the lint is -/// allowed HERE rather than workspace-wide so the reasoning stays attached to -/// the one site that does it. `read_volatile` on a composite gives no -/// per-field atomicity and no ordering, so a concurrent owner mutating its -/// counters can yield a snapshot whose fields disagree with each other. That -/// is accepted: these are `#[cfg(debug_assertions)]` diagnostic counters -/// (upstream's `MI_STAT` rule), nothing branches on them, and the alternative -/// — taking `&` to a struct another thread is writing — would be UB rather -/// than merely imprecise. The real fix is making the counters atomics, which -/// is the tracked follow-up (R-005 in the hardening audit). -#[allow( - clippy::volatile_composites, - reason = "racy diagnostic snapshot; see the doc comment. Tracked as R-005." -)] -pub fn for_each_heap(f: &mut dyn FnMut(&crate::heap::Heap)) { - heaps_lock(); - let mut hb = HEAPS_HEAD.load(Ordering::Acquire); - while !hb.is_null() { - // SAFETY: registry entries are live boxes. The read is unsynchronised - // with the owning thread's writes (see above) but stays within a live, - // correctly-aligned allocation for the whole of the copy. - unsafe { - let snapshot = core::ptr::read_volatile((*hb).heap.get()); - f(&snapshot); - hb = (*hb).next_box.load(Ordering::Acquire); - } - } - heaps_unlock(); -} - -/// Get (or create) the calling thread's heap box. NULL when creation fails -/// (memory exhaustion) — callers translate that into their own failure value -/// (a null allocation, a no-op collect); the TLS slot keeps the sentinel so a -/// later call retries. -#[inline] -pub fn heap_box() -> *mut HeapBox { - let hb = heap_tls::get(); - if hb == empty_heap_box_ptr() { - init_thread_heap() - } else { - hb - } -} - -/// This thread's heap box WITHOUT the created-yet test — may return the -/// shared empty sentinel. For the malloc fast path only. -/// -/// # Safety (for callers) -/// The returned box may be [`EMPTY_HEAP_BOX`], which is shared between every -/// uninitialised thread: callers must perform RAW-POINTER READS ONLY and must -/// never form a `&mut` (or write) through it. Route any miss to -/// [`ensure_heap`] before mutating anything. -#[inline] -pub fn heap_box_fast() -> *mut HeapBox { - heap_tls::get() -} - -/// Upgrade a possibly-sentinel box from [`heap_box_fast`] to this thread's -/// real, initialised heap box. NULL when creation fails (see [`heap_box`]). -#[inline] -pub fn ensure_heap(hb: *mut HeapBox) -> *mut HeapBox { - if hb == empty_heap_box_ptr() { - init_thread_heap() - } else { - hb - } -} - -/// Allocate + initialize a HeapBox (shared by thread bootstrap and -/// `mi_heap_new*`). Registered in the global registry; NOT installed in TLS. -/// -/// Returns NULL when the OS refuses the mapping (memory exhaustion). This -/// used to `expect()` — which, under the release profile's `panic = "abort"`, -/// turned OOM into a process abort on a path reachable from any fresh -/// thread's first `malloc`. The malloc contract on OOM is a null return, and -/// upstream returns NULL from `mi_heap_new` for the same reason (H-18/R-003 -/// in the hardening audit). -pub fn create_heap(tag: i32, allow_destroy: bool, arena_id: i32) -> *mut HeapBox { - // First-class heaps are not stored in `done_slot`. A thread that only - // ever called `mi_heap_new` / `heap_malloc` never ran `init_thread_heap`, - // so the TLS destructor never fired and leftover heaps stayed DELAYED - // (OH-rusty_alloc-13). Bootstrap the backing heap (and the exit hook) - // first. `init_thread_heap` calls `create_heap_uninstalled`, not this - // function, so there is no recursion. - if !crate::ONE_THREAD && heap_tls::get() == empty_heap_box_ptr() { - let _ = heap_box(); - } - create_heap_uninstalled(tag, allow_destroy, arena_id) -} - -/// Released heap boxes kept for the next heap, as upstream keeps -/// `mi_thread_data_t` (`TD_CACHE_SIZE`). A thread pool spawns and retires -/// threads constantly; without this, every thread's heap box was a fresh -/// `mmap` at its first allocation and a `munmap` at its exit, plus the -/// first-touch fault on the new mapping. A box in the cache is unregistered -/// and referenced by nothing (exactly the state in which it used to be -/// unmapped), and `create_heap_uninstalled` rewrites the whole struct. -static HEAP_BOX_CACHE: [AtomicPtr; 8] = [const { AtomicPtr::new(ptr::null_mut()) }; 8]; - -/// A cached heap box, or null. -fn heap_box_cache_take() -> *mut HeapBox { - for slot in &HEAP_BOX_CACHE { - if !slot.load(Ordering::Relaxed).is_null() { - let hb = slot.swap(ptr::null_mut(), Ordering::Acquire); - if !hb.is_null() { - return hb; - } - } - } - ptr::null_mut() -} - -/// Park a released heap box for reuse; false when the cache is full. -fn heap_box_cache_put(hb: *mut HeapBox) -> bool { - for slot in &HEAP_BOX_CACHE { - if slot - .compare_exchange(ptr::null_mut(), hb, Ordering::Release, Ordering::Relaxed) - .is_ok() - { - return true; - } - } - false -} - -fn create_heap_uninstalled(tag: i32, allow_destroy: bool, arena_id: i32) -> *mut HeapBox { - let size = core::mem::size_of::(); - // On a one-region target the FIRST descriptor is a static of the fixed - // backend's, so that a firmware's region is whole segments and needs no - // page carved out of it — the page is what made an aligned container of - // the region pad itself out to the next segment - // (`docs/plans/finished/region-alignment-bug.md` §7). Every later heap, - // and every heap on a hosted target, takes a page as before. - let first = if crate::ONE_REGION { - prim::fixed::take_first_heap_box() - } else { - None - }; - let hb: *mut HeapBox = match first { - Some(p) => p, - None => { - let cached = heap_box_cache_take(); - if cached.is_null() { - let Ok(block) = os::alloc_aligned(size, os::page_size(), true, false) else { - return ptr::null_mut(); - }; - block.ptr.cast() - } else { - cached - } - } - }; - // SAFETY: fresh committed mapping large enough for HeapBox; we initialize - // every field before publishing the pointer. - unsafe { - ptr::write( - hb, - HeapBox { - delayed: DelayedList::new(), - next_box: AtomicPtr::new(ptr::null_mut()), - owner_tid: thread_id(), - // Captured HERE, on the live creating thread, where the - // thread-local is unambiguously valid. - subproc: AtomicUsize::new(my_subproc()), - tag, - allow_destroy, - // A new heap starts as a copy of the empty one. On a - // one-region target that copy is taken from the sentinel - // itself, which already holds the whole template in flash; - // materialising `Heap::new()` here as well cost a second - // ~900-byte `.rodata` blob for the bin table. Hosted builds - // keep the constant, whose codegen was measured in Ir. - heap: UnsafeCell::new(if crate::ONE_REGION { - // SAFETY (within the enclosing block): a raw, - // non-overlapping read of the never-written sentinel's - // heap — the only access `heap_box_fast`'s contract - // permits, and `Heap` owns nothing that a bitwise copy - // could double-free. - ptr::read((*empty_heap_box_ptr()).heap.get()) - } else { - Heap::new() - }), - }, - ); - (*(*hb).heap.get()).delayed = &raw const (*hb).delayed; - (*(*hb).heap.get()).arena_id = arena_id; - (*(*hb).heap.get()).tag = tag; - // Per-heap CSPRNG stream (free-list keys, guarded sampling) -- seeded - // only where something draws from it. A build with neither `secure` - // nor guard pages has no consumer, and seeding still cost the ChaCha - // block function plus the splitmix fallback in flash. - if crate::RNG_USED { - (*(*hb).heap.get()).rng.reseed(); - } - // Guarded objects follow the option table (MI_GUARDED-equivalent) -- - // where a guard page can exist at all. `GUARD_PAGES` first, so that on - // a target without one the option reads fold away with the sampler - // (`docs/plans/finished/firmware-code-size.md`, lever 3). - let rate = crate::options::get(33).max(0) as usize; // guarded_sample_rate - let gmin = crate::options::get(30).max(0) as usize; // guarded_min - let gmax = crate::options::get(31).max(0) as usize; // guarded_max - if crate::GUARD_PAGES && gmax > 0 { - (*(*hb).heap.get()).guarded_set_size_bound(gmin, gmax); - (*(*hb).heap.get()) - .guarded_set_sample_rate(rate, crate::options::get(34).max(0) as usize); - } - } - heaps_register(hb); - hb -} - -#[cold] -fn init_thread_heap() -> *mut HeapBox { - let hb = create_heap_uninstalled(0, false, -1); - if hb.is_null() { - // OOM: install nothing — the TLS slot keeps the sentinel, so the - // NEXT allocation retries creation instead of dereferencing null. - return ptr::null_mut(); - } - heap_tls::set(hb); - BACKING_PTR.with(|c| c.set(hb)); - // The exit hook exists to abandon this heap when its thread ENDS. On a - // single-context build no thread ends, the backend's TLS never runs a - // destructor anyway (`prim::fixed::tls_new` discards it), and creating - // the slot was a compare-and-swap, a spin and a table entry paid once for - // nothing (`docs/plans/finished/firmware-code-size.md`, lever 2). - if !crate::ONE_THREAD { - done_slot().set(hb.cast::()); - } - hb -} - -ra_thread_local! { - /// The thread's original (backing) heap — what `mi_heap_get_backing` - /// returns regardless of `mi_heap_set_default` swaps. - static BACKING_PTR: Cell<*mut HeapBox> = const { Cell::new(ptr::null_mut()) }; -} - -/// `mi_heap_get_backing`. -pub fn backing_heap() -> *mut HeapBox { - let b = BACKING_PTR.with(|c| c.get()); - if b.is_null() { - let _ = heap_box(); // bootstraps and sets BACKING_PTR - BACKING_PTR.with(|c| c.get()) - } else { - b - } -} - -/// `mi_heap_set_default`: install `hb` as this thread's default; returns the -/// previous default. -/// -/// # Safety -/// `hb` must be a live heap owned by the calling thread. -pub unsafe fn set_default_heap(hb: *mut HeapBox) -> *mut HeapBox { - let prev = heap_box(); - heap_tls::set(hb); - prev -} - -/// The process-wide TLS slot whose destructor abandons a dying thread's heap. -fn done_slot() -> prim::TlsSlot { - static RAW: AtomicUsize = AtomicUsize::new(0); - static INIT: AtomicBool = AtomicBool::new(false); - let raw = RAW.load(Ordering::Acquire); - if raw != 0 { - // SAFETY: RAW holds an into_raw of a slot created below. - return unsafe { prim::TlsSlot::from_raw(raw - 1) }; - } - if INIT - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - { - // A panic here would be wrong twice over. An allocator must not unwind - // into its C callers (which is why the release profile is - // panic=abort), AND the winner of this race is the only thread that - // ever publishes RAW — so unwinding past the store below would strand - // every other thread in the spin loop FOREVER, turning a rare - // resource failure into a silent process-wide hang. Fail loudly and - // deterministically instead, identically in debug and release. - let Some(slot) = prim::TlsSlot::new(Some(thread_done_cb)) else { - crate::abort(); - }; - RAW.store(slot.into_raw() + 1, Ordering::Release); // +1: 0 is the unset sentinel - } - loop { - let raw = RAW.load(Ordering::Acquire); - if raw != 0 { - // SAFETY: as above. - return unsafe { prim::TlsSlot::from_raw(raw - 1) }; - } - core::hint::spin_loop(); - } -} - -#[cfg(all(windows, not(miri)))] -unsafe extern "system" fn thread_done_cb(v: *const c_void) { - if !v.is_null() { - // SAFETY: v is the HeapBox this thread stored in the done slot. - unsafe { thread_done(v.cast_mut().cast()) }; - } -} - -#[cfg(any(not(windows), miri))] -unsafe extern "C" fn thread_done_cb(v: *mut c_void) { - if !v.is_null() { - // SAFETY: v is the HeapBox this thread stored in the done slot. - unsafe { thread_done(v.cast()) }; - } -} - -/// Abandon the dying thread's heap. Teardown order is the loom-modeled one: -/// (1) every surviving page → NEVER (spins out FREEING remotes), (2) THEN -/// drain the delayed list, (3) THEN release the heap storage. -/// -/// # Safety -/// Must run on the dying thread with `hb` its live heap box, exactly once. -pub unsafe fn thread_done(hb: *mut HeapBox) { - // SAFETY: per contract `hb` is this thread's live box; `owner_tid` is a - // plain field. - let tid = unsafe { (*hb).owner_tid }; - // SAFETY: per contract we are the owner and this runs exactly once; - // sequencing per the model. - unsafe { thread_done_one(hb) }; - // Back to the SENTINEL, not null — the slot is never null, so a - // post-teardown malloc re-enters the generic path and re-initialises. - heap_tls::set(empty_heap_box_ptr()); - // First-class heaps this thread created are not stored in `done_slot` - // (`set_default_heap` does not update it). Leaving them DELAYED meant a - // second remote free after thread exit returned (OH-rusty_alloc-13). - // Abandon every remaining heap we still own so pages go NEVER. Iterative: - // a dying thread can own many first-class heaps; do not recurse. - if !crate::ONE_THREAD { - // Stall guard: a dying thread that keeps re-initialising (empty TLS - // malloc) must not loop forever. Same order as `collect_inner`'s - // `MAX_RECLAIM`. - const MAX_OWNED_HEAPS: usize = 1 << 20; - for _ in 0..MAX_OWNED_HEAPS { - let extra = take_heap_owned_by(tid); - if extra.is_null() { - break; - } - // SAFETY: extra is a live box this dying thread still owns. - unsafe { thread_done_one(extra) }; - } - } -} - -/// Abandon one heap box (NEVER + delayed drain + unregister + free the box). -/// -/// # Safety -/// `hb` is a live heap box owned by the calling thread, not used again. -unsafe fn thread_done_one(hb: *mut HeapBox) { - // SAFETY: per contract we are the owner; sequencing per the model. - unsafe { - // Read the subprocess tag from the BOX, not from `my_subproc()`. We are - // inside a platform TLS destructor and Rust's `thread_local!` storage - // may already be torn down — on aarch64-apple-darwin it reads back 0, - // which silently dropped every tagged thread's segments into the main - // subprocess. Clamped because a corrupt tag must not index out of - // bounds on the teardown path. - let sp = (*hb).subproc.load(Ordering::Acquire).min(MAX_SUBPROCS - 1); - let h = &mut *(*hb).heap.get(); - // Retire everything that is already dead (also drains delayed once). - // TEARDOWN variant: must NOT adopt orphans into a heap we are about to - // destroy — that was the 0.3.1 segfault. - h.collect_for_teardown(); - // Walk remaining segments: transition pages to NEVER, disown, publish. - let mut seg = h.segments; - h.segments = ptr::null_mut(); - while !seg.is_null() { - let next = (*seg).next; - let end = (*seg).next_free_slice as usize; - let mut idx = segment::HEADER_SLICES; - // Running pointer, not `pages[idx]`: `Page` is not a power-of-two - // size, so indexing costs a multiply plus a bounds check on every - // carved slice of every segment a dying thread owns. - let base: *mut crate::page::Page = (&raw mut (*seg).pages).cast(); - let mut slot: *mut crate::page::Page = base.add(idx); - while idx < end { - let len = ((*slot).slice_count as usize).max(1); - if (*slot).block_size > 0 { - // ONE locked read-modify-write, not two: taking the - // cross-thread chain and publishing NEVER are the same - // word, and doing them separately paid two CAS loops per - // live page of every segment a dying thread owns. - page_collect_and_set_flag(slot, XFLAG_NEVER); - (*slot).xheap.store(0, Ordering::Release); - } - idx += len; - // `wrapping_add`, not `add`: the pointer is only ever - // DEREFERENCED while `idx < end` holds, which the span-partition - // invariant makes in-bounds — but the final advance of the loop - // computes a pointer that may be one past the carved region, and - // forming that with `add` would be UB even unread. - slot = slot.wrapping_add(len); - } - if (*seg).used_pages == 0 { - let _ = segment::segment_free(seg); - h.stats.segments_freed += 1; - } else { - (*seg).thread_id.store(0, Ordering::Release); - abandoned_push(seg, sp); - } - seg = next; - } - // Huge segments: same teardown shape (collect NEVER-bound frees, - // release dead ones, abandon the rest). - let mut hseg = h.huge_segments; - h.huge_segments = ptr::null_mut(); - while !hseg.is_null() { - let next = (*hseg).next; - let pg = &raw mut (*hseg).pages[1]; - page_collect(pg); - if (*pg).used == 0 { - let _ = segment::huge_free(hseg); - h.stats.segments_freed += 1; - } else { - page_set_flag(pg, XFLAG_NEVER); - (*pg).xheap.store(0, Ordering::Release); - (*hseg).thread_id.store(0, Ordering::Release); - abandoned_push(hseg, sp); - } - hseg = next; - } - // Post-NEVER drain: blocks a FREEING remote landed while we walked. - // Their pages are abandoned now, so free_local would be wrong — but - // these blocks belong to pages we JUST abandoned; push them back on - // their pages' own lists (flag NEVER routes them there). - let mut b = (*hb).delayed.head.swap(0, Ordering::AcqRel) as *mut crate::page::Block; - while !b.is_null() { - let next = (*b).next; - let seg = segment::segment_of(b.cast()); - let pg = segment::page_of(seg, b.cast()); - crate::page::remote_free(pg, b); - b = next; - } - // Release the heap storage and clear the fast-path pointer. The - // first descriptor on a one-region target is a static, not a page: - // nothing to return (and this path is unreachable there anyway). - heaps_unregister(hb); - // A static first box is not a page and is never cached; any other - // box is cached when there is room, and unmapped otherwise. - let kept = - (crate::ONE_REGION && prim::fixed::is_first_heap_box(hb)) || heap_box_cache_put(hb); - if !kept { - let size = core::mem::size_of::(); - let blockdesc = os::OsBlock { - ptr: hb.cast(), - size: os::page_align_up(size), - is_large: false, - is_zero: false, - }; - let _ = os::free(blockdesc); - } - } -} - -/// `mi_heap_delete`: free the heap structure, MIGRATING its live pages and -/// segments into the thread's backing heap (blocks stay valid). -/// -/// # Safety -/// `hb` must be a live first-class heap owned by the calling thread, not the -/// backing heap, used never again. -pub unsafe fn heap_delete(hb: *mut HeapBox) { - let backing = backing_heap(); - if hb == backing { - return; // deleting the backing heap is a no-op (upstream-compatible) - } - // SAFETY: owner thread per contract; absorb everything into backing. - unsafe { - debug_assert_eq!((*hb).owner_tid, thread_id()); - let h = &mut *(*hb).heap.get(); - let bh = &mut *(*backing).heap.get(); - // Teardown: this heap is being deleted, so it must not adopt orphans. - h.collect_for_teardown(); - let mut seg = h.segments; - h.segments = ptr::null_mut(); - while !seg.is_null() { - let next = (*seg).next; - // `next` is read BEFORE adopting and `seg` is not touched after, so - // a release during adoption is safe here. - let _ = bh.adopt_segment(seg); // re-homes queues + xheap (tid ours) - seg = next; - } - let mut hseg = h.huge_segments; - h.huge_segments = ptr::null_mut(); - while !hseg.is_null() { - let next = (*hseg).next; - let pg = &raw mut (*hseg).pages[1]; - (*pg).xheap.store(bh.delayed as usize, Ordering::Release); - (*hseg).next = bh.huge_segments; - bh.huge_segments = hseg; - hseg = next; - } - // Fold the dying heap's counters into backing so process totals hold. - merge_stats(&mut bh.stats, &h.stats); - release_heap_box(hb); - } -} - -/// `mi_heap_destroy`: drop EVERY block of the heap at once and release its -/// memory. Falls back to delete semantics when the heap was created with -/// `allow_destroy = false`. -/// -/// # Safety -/// As [`heap_delete`], and additionally no block of this heap may be used -/// afterwards (the C contract). -pub unsafe fn heap_destroy(hb: *mut HeapBox) { - // SAFETY: owner thread per contract. - unsafe { - if !(*hb).allow_destroy { - heap_delete(hb); - return; - } - debug_assert_eq!((*hb).owner_tid, thread_id()); - let h = &mut *(*hb).heap.get(); - // Spin out any in-flight FREEING remotes before the box dies; blocks - // they queued are dying with the heap, so the delayed list is simply - // discarded. - let mut seg = h.segments; - while !seg.is_null() { - let end = (*seg).next_free_slice as usize; - let mut idx = segment::HEADER_SLICES; - // Same running-pointer walk as `thread_done`'s — the sibling site, - // checked because a defect in one slice walk is a defect in every - // slice walk. Not exercised by `mleak` (this is `heap_destroy`, not - // thread teardown), so it carries no measurement of its own; it is - // the identical transformation on identical code. - let base: *mut crate::page::Page = (&raw mut (*seg).pages).cast(); - let mut slot: *mut crate::page::Page = base.add(idx); - while idx < end { - let len = ((*slot).slice_count as usize).max(1); - if (*slot).block_size > 0 { - page_set_flag(slot, XFLAG_NEVER); - (*slot).xheap.store(0, Ordering::Release); - } - idx += len; - slot = slot.wrapping_add(len); - } - let next = (*seg).next; - let _ = segment::segment_free(seg); - seg = next; - } - let mut hseg = h.huge_segments; - while !hseg.is_null() { - let pg = &raw mut (*hseg).pages[1]; - page_set_flag(pg, XFLAG_NEVER); - (*pg).xheap.store(0, Ordering::Release); - let next = (*hseg).next; - let _ = segment::huge_free(hseg); - hseg = next; - } - release_heap_box(hb); - } -} - -fn merge_stats(into: &mut crate::heap::Stats, from: &crate::heap::Stats) { - into.allocs += from.allocs; - into.frees += from.frees; - into.generic += from.generic; - into.pages_fresh += from.pages_fresh; - into.segments += from.segments; - into.huge_allocs += from.huge_allocs; - into.extends += from.extends; - into.large_allocs += from.large_allocs; - into.pages_retired += from.pages_retired; - into.segments_freed += from.segments_freed; - into.realloc_in_place += from.realloc_in_place; - into.realloc_moved += from.realloc_moved; - into.delayed_frees += from.delayed_frees; - into.reclaims += from.reclaims; -} - -/// Unregister and release a HeapBox, repairing the TLS default if needed. -/// -/// # Safety -/// `hb` live, owned by the calling thread, all contents already migrated or -/// released. -unsafe fn release_heap_box(hb: *mut HeapBox) { - heaps_unregister(hb); - if heap_tls::get() == hb { - heap_tls::set(backing_heap()); - } - // A first-class heap created before the first allocation would have been - // handed the fixed backend's static descriptor; that is not a page and - // must not be returned to the region. - if crate::ONE_REGION && prim::fixed::is_first_heap_box(hb) { - return; - } - if heap_box_cache_put(hb) { - return; - } - let size = core::mem::size_of::(); - let blockdesc = os::OsBlock { - ptr: hb.cast(), - size: os::page_align_up(size), - is_large: false, - is_zero: false, - }; - // SAFETY: hb's mapping came from create_heap's os::alloc_aligned with this - // exact size; unregistered above, no references remain. - unsafe { - let _ = os::free(blockdesc); - } -} - -// --------------------------------------------------------------------------- -// Global abandoned-segment list (spin-locked; cold path). -// --------------------------------------------------------------------------- - -/// Subprocess isolation (M6, plan §5.9): threads tagged with a subproc id -/// abandon/reclaim only within it — separate interpreters never exchange -/// segments. Id 0 is the main subprocess. -const MAX_SUBPROCS: usize = 64; - -static ABANDONED_LOCK: AtomicBool = AtomicBool::new(false); -static ABANDONED_HEADS: [AtomicPtr; MAX_SUBPROCS] = - [const { AtomicPtr::new(ptr::null_mut()) }; MAX_SUBPROCS]; -static SUBPROC_NEXT: AtomicUsize = AtomicUsize::new(1); -/// Diagnostic: segments currently abandoned (all subprocs). -pub static ABANDONED_COUNT: AtomicUsize = AtomicUsize::new(0); - -ra_thread_local! { - static SUBPROC: Cell = const { Cell::new(0) }; -} - -/// `mi_subproc_main` → 0. -pub fn subproc_main() -> usize { - 0 -} - -/// `mi_subproc_new`: allocate a fresh subprocess id. -/// -/// Exhaustion is an invalid id (`MAX_SUBPROCS`), not an assert (OH-rusty_alloc-21). -pub fn subproc_new() -> usize { - loop { - let id = SUBPROC_NEXT.load(Ordering::Acquire); - if id >= MAX_SUBPROCS { - return MAX_SUBPROCS; - } - if SUBPROC_NEXT - .compare_exchange_weak(id, id + 1, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - { - return id; - } - } -} - -/// `mi_subproc_delete`: ids are not recycled in v1; abandoned segments of the -/// subproc are drained into the main subproc so they stay reclaimable. -pub fn subproc_delete(id: usize) { - if id == 0 || id >= MAX_SUBPROCS { - return; - } - abandoned_lock(); - // SAFETY: list surgery under the lock. - unsafe { - let mut seg = ABANDONED_HEADS[id].swap(ptr::null_mut(), Ordering::AcqRel); - while !seg.is_null() { - let next = (*seg).next; - (*seg).next = ABANDONED_HEADS[0].load(Ordering::Relaxed); - ABANDONED_HEADS[0].store(seg, Ordering::Relaxed); - seg = next; - } - } - abandoned_unlock(); -} - -/// `mi_subproc_add_current_thread`. -pub fn subproc_add_current_thread(id: usize) { - if id >= MAX_SUBPROCS { - return; - } - SUBPROC.with(|c| c.set(id)); - // Mirror onto this thread's backing heap so TEARDOWN can read the tag - // without touching the thread-local (see `HeapBox::subproc`). Tagging - // before the first allocation is the common order and is covered by - // `create_heap` reading `my_subproc()`; this covers tagging AFTER the heap - // already exists. Deliberately does NOT bootstrap a heap — a thread that - // tags itself and never allocates should not be given one. - let hb = BACKING_PTR.with(|c| c.get()); - if !hb.is_null() { - // SAFETY: hb is this thread's own live backing box; `subproc` is - // atomic, so the teardown read needs no other synchronisation. - unsafe { (*hb).subproc.store(id, Ordering::Release) }; - } -} - -fn my_subproc() -> usize { - SUBPROC.with(|c| c.get()) -} - -/// `mi_abandoned_visit_blocks`: visit live pages/blocks of a subprocess's -/// abandoned segments, holding the list lock (segments are pinned meanwhile). -pub fn abandoned_visit_blocks( - subproc_id: usize, - heap_tag: i32, - visit_blocks: bool, - f: &mut dyn FnMut(&crate::heap::AreaInfo, *mut u8, usize) -> bool, -) -> bool { - if subproc_id >= MAX_SUBPROCS { - return false; - } - abandoned_lock(); - let mut ok = true; - let mut seg = ABANDONED_HEADS[subproc_id].load(Ordering::Acquire); - while !seg.is_null() && ok { - // SAFETY: the lock pins abandoned segments (adopt also takes it). - unsafe { - ok = crate::heap::visit_segment_blocks(seg, heap_tag, visit_blocks, false, f); - seg = (*seg).next; - } - } - abandoned_unlock(); - ok -} - -fn abandoned_lock() { - while ABANDONED_LOCK - .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) - .is_err() - { - core::hint::spin_loop(); - } -} - -fn abandoned_unlock() { - ABANDONED_LOCK.store(false, Ordering::Release); -} - -/// Publish `seg` to subprocess `sp`'s abandoned list. -/// -/// `sp` is passed in rather than read from `my_subproc()`: every caller is on -/// the teardown path, where the `SUBPROC` thread-local may already be gone (see -/// `HeapBox::subproc`). The owning heap box carries the tag instead. -fn abandoned_push(seg: *mut Segment, sp: usize) { - // The correctness argument behind `abandoned_pop`'s early return: if this - // ever runs on a single-context build, a thread ended, and the target's - // single-thread assertion was false all along. Worth knowing for its own - // sake -- `SingleThreadCell` and `prim::fixed` rest on the same claim. - if crate::ONE_THREAD { - unreachable!("a segment was abandoned on a build that asserted a single thread"); - } - debug_assert!(sp < MAX_SUBPROCS); - // PURGE BEFORE ORPHANING (`abandoned_page_purge`, on by default). Until - // some other thread adopts this segment it has no owner, so every page it - // touched stays resident for an unbounded time and nothing can reuse it. - // Done here, before publishing to the list, because this is the last - // instant the caller still exclusively owns the segment. - if crate::options::is_enabled(30) { - // SAFETY: still owned by the dying thread; free spans hold no live - // blocks, and each purged span is marked so reuse re-commits it. - unsafe { crate::segment::purge_free_spans(seg) }; - } - abandoned_lock(); - // SAFETY: seg is disowned; its `next` is ours to use as the list link. - unsafe { - (*seg).next = ABANDONED_HEADS[sp].load(Ordering::Relaxed); - } - ABANDONED_HEADS[sp].store(seg, Ordering::Relaxed); - ABANDONED_COUNT.fetch_add(1, Ordering::Relaxed); - abandoned_unlock(); -} - -/// Pop one abandoned segment from the CALLER's subprocess and take ownership. -/// Returns null when none are available. -pub fn abandoned_pop() -> *mut Segment { - // A single-context build has nothing to pop, by construction: a segment is - // only ever abandoned by a thread that ENDS (`thread_done`), and the - // target has asserted there is exactly one thread for the life of the - // program. Folding this to null is what makes `adopt_segment` unreachable - // from the allocation path, so the linker drops it -- 1,154 bytes on an - // ESP32-S3 that could never have run (`docs/plans/finished/firmware-code-size.md`, - // lever 1). `heap_delete` still calls it directly, and a firmware that - // deletes first-class heaps keeps it; that is a real path, not a - // cross-thread one. - if crate::ONE_THREAD { - return ptr::null_mut(); - } - let sp = my_subproc(); - if ABANDONED_HEADS[sp].load(Ordering::Acquire).is_null() { - return ptr::null_mut(); // fast no-lock exit for the common case - } - abandoned_lock(); - let seg = ABANDONED_HEADS[sp].load(Ordering::Relaxed); - if !seg.is_null() { - // SAFETY: list links are protected by the lock. - unsafe { - ABANDONED_HEADS[sp].store((*seg).next, Ordering::Relaxed); - (*seg).next = ptr::null_mut(); - (*seg).thread_id.store(thread_id(), Ordering::Release); - } - ABANDONED_COUNT.fetch_sub(1, Ordering::Relaxed); - } - abandoned_unlock(); - seg -} +//! Thread + heap lifecycle (mirrors upstream `init.c` + the abandonment side +//! of `arena-abandon.c`). +//! +//! Every thread gets a lazily-created [`HeapBox`] (one OS page from the prim +//! layer — never from the allocator itself, so bootstrap can't recurse). The +//! fast path reaches it through a const-init `thread_local!` `Cell` pointer, +//! which the R1 spike measured at atomic-load parity (0.3 ns) — no dtor is +//! registered on that key (Cell is !Drop), so accessing it never allocates. +//! +//! Thread EXIT runs through the prim TLS destructor (`FlsAlloc` / +//! `pthread_key_create`, built in M1 for exactly this): drain delayed frees, +//! retire what died, transition every surviving page to `NEVER` (spinning out +//! in-flight `FREEING` remotes — the loom-modeled teardown order), zero the +//! segments' thread id, and publish them on the global abandoned list. Any +//! other thread's allocation slow path adopts them later. + +use core::cell::{Cell, UnsafeCell}; +use core::ffi::c_void; +use core::ptr; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +use crate::heap::Heap; +use crate::page::{ + DelayedList, XFLAG_NEVER, page_collect, page_collect_and_set_flag, page_set_flag, +}; +use crate::segment::{self, Segment}; +use crate::{os, prim}; + +/// A heap plus the cross-thread fields remote threads may touch. Remote +/// threads only ever dereference `delayed` (via page `xheap` addresses) — +/// never the heap itself — which is what makes `&mut` on the inner heap sound. +#[repr(C)] +pub struct HeapBox { + /// Cross-thread delayed-free list (remote-reachable). MUST stay the first + /// field: pages store `&delayed` in `xheap`, and owner routing recovers + /// the box by container-of (delayed at offset 0 ⇒ the addresses coincide). + pub delayed: DelayedList, + /// Next box in the global registry (guarded by the registry lock). + /// AtomicPtr, not usize: reachability (miri's leak check and honesty + /// about what is cached vs leaked) follows pointers, not integers. + pub next_box: AtomicPtr, + /// Owning thread id (heaps allocate only from their creating thread). + pub owner_tid: usize, + /// The subprocess this heap's segments are abandoned into. + /// + /// Mirrored from the `SUBPROC` thread-local so that TEARDOWN never has to + /// read it. `thread_done` runs from a platform TLS destructor + /// (`pthread_key_create` / `FlsAlloc`), and the destruction order between + /// *that* callback and Rust's own `thread_local!` storage is unspecified on + /// every platform. On aarch64-apple-darwin it is observably wrong: + /// `my_subproc()` reads back 0 inside `thread_done`, so a tagged thread's + /// segments landed in the MAIN subprocess and `mi_subproc_*` isolation was + /// silently lost — the abandon fired correctly, only its destination was + /// wrong. The heap box is our own mapping and is alive for the whole of + /// teardown (it is the value passed *to* the destructor), so reading the tag + /// from here is correct on every target instead of accidentally correct on + /// one. + pub subproc: AtomicUsize, + /// Heap tag (`mi_heap_new_ex`; reported by the block visitor). + pub tag: i32, + /// Whether `mi_heap_destroy` may drop live blocks (else it delegates to + /// delete semantics). + pub allow_destroy: bool, + /// The owner-only heap state. + pub heap: UnsafeCell, +} + +const _: () = assert!(core::mem::offset_of!(HeapBox, delayed) == 0); + +/// Wrapper so the empty sentinel box can be a `static`. +#[repr(transparent)] +pub struct EmptyHeapBox(HeapBox); + +// SAFETY: the sentinel is never written and never escapes to remote threads: +// no page ever stores its `delayed` address in `xheap` (pages are re-homed +// only by REAL heaps), its `owner_tid` of 0 matches no live thread, and every +// reader goes through raw-pointer READS — `heap_box_fast` documents that no +// `&mut` may ever be formed to it. It exists purely so the malloc fast path +// has a valid heap to read before this thread has created one. +unsafe impl Sync for EmptyHeapBox {} + +/// The "no heap yet" sentinel: every `direct` slot holds the empty PAGE +/// sentinel, so a fresh thread's first malloc falls through to the generic +/// path — which detects this box and performs the real thread init — without +/// the fast path ever testing for it. This is upstream's `_mi_heap_empty` +/// trick, and the exact design our own [`crate::page::EMPTY_PAGE`] already +/// uses one level down: replace a null-test on the hot path with a sentinel +/// whose data routes the miss to the slow path for free. +/// +/// `export_name` (not mangled) because the x86-64 Linux TLS slot's `.tdata` +/// initializer below names it in assembly; `.hidden` there keeps it out of +/// the cdylib's export table. +/// +/// **In flash on a one-region target.** The sentinel is read and never +/// written — `heap_box_fast` documents that contract, the fast path only +/// writes after a non-null pop and every `direct` slot here points at the +/// empty page whose free list is null, and no page ever carries its address +/// in `xheap`. Its type has interior mutability (a `DelayedList`, an +/// `AtomicPtr`, the `UnsafeCell`), which is why rustc places it in +/// `.data` by default: 1,752 bytes paid once in flash for the image and once +/// more in RAM for the loader's copy, the largest single item left on the +/// ESP32-S3 after 2.0.2 and RAM the linker takes from the stack. Placing it +/// in `.rodata.*` keeps the flash copy and drops the RAM one. Any write +/// would fault loudly against the flash cache, which is the right way for +/// the contract to fail. Bare metal only — a hosted target may need to +/// relocate the pointers inside it at load, and has RAM to spare. +#[cfg_attr( + all( + ra_single_threaded, + not(miri), + not(windows), + not(unix), + not(target_arch = "wasm32") + ), + unsafe(link_section = ".rodata.rusty_alloc_empty_heap_box") +)] +#[unsafe(export_name = "__ra_empty_heap_box")] +pub static EMPTY_HEAP_BOX: EmptyHeapBox = EmptyHeapBox(HeapBox { + delayed: DelayedList::new(), + next_box: AtomicPtr::new(ptr::null_mut()), + owner_tid: 0, + subproc: AtomicUsize::new(0), + tag: 0, + allow_destroy: false, + heap: UnsafeCell::new(Heap::new()), +}); + +/// Pointer to the shared empty heap box. +#[inline] +pub const fn empty_heap_box_ptr() -> *mut HeapBox { + (&raw const EMPTY_HEAP_BOX.0).cast_mut() +} + +/// Recover the owning HeapBox from a page's `xheap` value (the container-of +/// trick over the offset-0 delayed list). +/// +/// # Safety +/// `xheap` must be a live HeapBox's delayed-list address (nonzero). +#[inline] +pub unsafe fn box_of_xheap(xheap: usize) -> *mut HeapBox { + debug_assert!(xheap != 0); + xheap as *mut HeapBox +} + +// Global registry of live heap boxes: keeps every heap (and through it every +// segment) reachable-from-static — the process-wide stats walk of M7, and +// what makes miri's reachability-based leak check reflect reality (an +// allocator's arenas at process exit are cached state, not leaks). +static HEAPS_LOCK: AtomicBool = AtomicBool::new(false); +static HEAPS_HEAD: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +fn heaps_lock() { + while HEAPS_LOCK + .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + core::hint::spin_loop(); + } +} + +fn heaps_unlock() { + HEAPS_LOCK.store(false, Ordering::Release); +} + +fn heaps_register(hb: *mut HeapBox) { + heaps_lock(); + // SAFETY: hb is fully initialized and not yet published. + unsafe { + (*hb) + .next_box + .store(HEAPS_HEAD.load(Ordering::Relaxed), Ordering::Relaxed); + } + HEAPS_HEAD.store(hb, Ordering::Relaxed); + heaps_unlock(); +} + +fn heaps_unregister(hb: *mut HeapBox) { + heaps_lock(); + let mut cur = HEAPS_HEAD.load(Ordering::Relaxed); + if cur == hb { + // SAFETY: links guarded by the lock. + unsafe { + HEAPS_HEAD.store((*hb).next_box.load(Ordering::Relaxed), Ordering::Relaxed); + } + } else { + while !cur.is_null() { + // SAFETY: links guarded by the lock; nodes are live boxes. + unsafe { + let nxt = (*cur).next_box.load(Ordering::Relaxed); + if nxt == hb { + (*cur) + .next_box + .store((*nxt).next_box.load(Ordering::Relaxed), Ordering::Relaxed); + break; + } + cur = nxt; + } + } + } + heaps_unlock(); +} + +/// Unlink and return one live heap box owned by `tid`, or null. +/// +/// The lock is released before return so the caller can abandon the heap +/// without re-entering `HEAPS_LOCK` (abandon walks pages and can allocate). +fn take_heap_owned_by(tid: usize) -> *mut HeapBox { + heaps_lock(); + let mut prev: *mut HeapBox = ptr::null_mut(); + let mut cur = HEAPS_HEAD.load(Ordering::Relaxed); + while !cur.is_null() { + // SAFETY: links and boxes are live under the lock. + unsafe { + let nxt = (*cur).next_box.load(Ordering::Relaxed); + if (*cur).owner_tid == tid { + if prev.is_null() { + HEAPS_HEAD.store(nxt, Ordering::Relaxed); + } else { + (*prev).next_box.store(nxt, Ordering::Relaxed); + } + heaps_unlock(); + return cur; + } + prev = cur; + cur = nxt; + } + } + heaps_unlock(); + ptr::null_mut() +} + +/// Storage for the calling thread's default-heap pointer. +/// +/// On x86-64 Linux this is a RAW ELF TLS symbol reached with the +/// **initial-exec** model rather than `thread_local!`'s general-dynamic. That +/// was the last `__tls_get_addr` call on the malloc fast path — 7.3 M +/// instructions on perl, a third of everything still separating us from +/// mimalloc after the M10b free-path bricks. Stable Rust has no +/// `-Z tls-model`, so the symbol is declared in `global_asm!` and the linker +/// resolves its offset. +/// +/// **Why this is sound where a thread-pointer-keyed side table is not.** The +/// storage IS the thread's own TLS block. Every thread gets a fresh block +/// initialised from the `.tdata` template (the empty-heap sentinel) when it +/// is created, so a recycled TCB cannot hand a new thread a dead thread's +/// heap — the staleness +/// question that made the keyed-cache design a P0 risk simply does not arise +/// here. What initial-exec costs instead is a LOAD-TIME constraint: it needs a +/// slot in the static TLS block, so a `dlopen` long after startup could fail +/// to load us. That fails loudly at load time rather than corrupting memory, +/// and it is the same trade upstream mimalloc ships +/// (`__attribute__((tls_model("initial-exec")))` on `_mi_heap_default`). +#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] +mod heap_tls { + use super::HeapBox; + + // An 8-byte thread-local slot. `.tdata` (not `.tbss`): the slot is + // INITIALISED to the empty-heap sentinel, so every new thread reads a + // valid heap pointer from its very first malloc and the fast path needs + // no "not yet created" test at all — the sentinel's empty direct table + // routes the first allocation to the generic path, which performs the + // real init there (upstream's `_mi_heap_empty` design). The loader + // relocates the `.quad` in the TLS template once at load time; every + // thread's block is then copied from that template. `.hidden` makes both + // symbols non-preemptible, which is what permits the initial-exec + // (GOTTPOFF) relocation instead of a general-dynamic call, and keeps the + // sentinel out of the cdylib's export table. + core::arch::global_asm!( + ".section .tdata,\"awT\",@progbits", + ".globl __ra_tls_heap", + ".hidden __ra_tls_heap", + ".hidden __ra_empty_heap_box", + ".type __ra_tls_heap,@object", + ".p2align 3", + "__ra_tls_heap:", + ".quad __ra_empty_heap_box", + ".size __ra_tls_heap,8", + ".text", + ); + + /// Address of THIS thread's slot: the thread pointer plus the offset the + /// dynamic linker wrote into the GOT. + #[inline(always)] + fn slot() -> *mut *mut HeapBox { + let off: usize; + // SAFETY: reads the GOT entry the dynamic linker relocated at load + // time, before any of our code can run, and never writes. It is + // constant for the life of the process, hence `pure` + `readonly` — + // that is what lets LLVM hoist and CSE the address computation + // instead of treating the block as an optimisation barrier. + unsafe { + core::arch::asm!( + "mov {o}, qword ptr [rip + __ra_tls_heap@GOTTPOFF]", + o = out(reg) off, + options(nostack, preserves_flags, readonly, pure), + ); + } + // `thread_id()` IS the fs base on this target. TLS offsets are + // negative there (the block grows down from the thread pointer), so + // this adds a two's-complement offset and must wrap. + core::ptr::with_exposed_provenance_mut(super::thread_id().wrapping_add(off)) + } + + /// Read this thread's heap pointer in TWO instructions. + /// + /// The obvious form — `slot().read()` — costs FOUR: load the offset from + /// the GOT, read the thread pointer from `fs:0`, add them, dereference. + /// But x86 performs that addition IN THE ADDRESSING MODE: `fs:[reg]` is a + /// segment-relative load, so the explicit `fs:0` read and the add both + /// disappear. Measured at 4.00 Ir per malloc before this change — a + /// quarter of our entire malloc-side deficit spent just locating the heap. + #[inline(always)] + pub fn get() -> *mut HeapBox { + let v: *mut HeapBox; + // SAFETY: reads the GOT entry the loader relocated at startup, then + // loads 8 aligned bytes of this thread's own TLS block through the fs + // segment. Both are reads; nothing is written. + unsafe { + core::arch::asm!( + "mov {t}, qword ptr [rip + __ra_tls_heap@GOTTPOFF]", + "mov {o}, qword ptr fs:[{t}]", + t = out(reg) _, + o = out(reg) v, + options(nostack, preserves_flags, readonly), + ); + } + v + } + + #[inline(always)] + pub fn set(hb: *mut HeapBox) { + // SAFETY: as `get` — the slot is owned exclusively by this thread. + unsafe { slot().write(hb) } + } +} + +/// Every other target keeps the `thread_local!`: Windows TLS is an index into +/// a per-thread array (no `__tls_get_addr` to remove), and miri models the +/// macro but not a hand-rolled TLS symbol. +#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))] +mod heap_tls { + use super::HeapBox; + use core::cell::Cell; + + ra_thread_local! { + /// Fast-path heap pointer. Const-init + !Drop ⇒ plain TLS access, no + /// lazy-init branch, no allocation ever. Initialised to the + /// empty-heap SENTINEL (never null) so the malloc fast path can read + /// through it unconditionally; see `EMPTY_HEAP_BOX`. + static HEAP_PTR: Cell<*mut HeapBox> = const { Cell::new(super::empty_heap_box_ptr()) }; + } + + #[inline(always)] + pub fn get() -> *mut HeapBox { + HEAP_PTR.with(|c| c.get()) + } + + #[inline(always)] + pub fn set(hb: *mut HeapBox) { + HEAP_PTR.with(|c| c.set(hb)); + } +} + +ra_thread_local! { + /// Cached OS thread id. `free` needs the calling thread's id on EVERY + /// call to route local-vs-remote; the raw `prim::thread_id()` is a libc + /// call (`pthread_self` through the PLT from a cdylib) measured at + /// 1.41 ns vs 0.25 ns for this cache — ~18-20% of a whole malloc+free + /// pair (M9 probe, both platforms). 0 = not yet cached; no real thread id + /// is 0 on either platform. + static TID: Cell = const { Cell::new(0) }; +} + +/// The calling thread's id — read STRAIGHT FROM THE THREAD-POINTER REGISTER +/// where the target allows it. +/// +/// Why this and not `thread_local!`: the shipping artifact is a **cdylib** +/// (LD_PRELOAD). Rust's `thread_local!` in a shared library compiles to the +/// general-dynamic TLS model, so every access is a CALL into `ld.so`'s +/// `__tls_get_addr` — the per-function instruction profile put that call at +/// 12.97 M Ir, 1.96% of a whole lua run, comparable to our entire `free`. +/// (The earlier ns-level probe missed it because it measured TLS inside an +/// EXECUTABLE, where the model is local-exec: a register offset, no call. +/// Measure the artifact you ship.) +/// +/// A ceiling probe with `-Z tls-model=initial-exec` valued removing those +/// calls at 2.00% of our instructions and 33% of the gap to mimalloc; this +/// gets the same effect on stable, for the id, with no linker involvement — +/// the thread pointer is one register read. It is exactly what mimalloc's +/// `_mi_thread_id()` does. +/// +/// Uniqueness: the value is the thread-control-block address, unique among +/// LIVE threads. It can be recycled after a thread dies, which is sound here +/// for the same reason it is in mimalloc: a dying thread abandons its +/// segments (`thread_id` stored as 0) before its TCB can be reused, so a +/// recycled id can never match a stale segment. +#[inline(always)] +pub fn thread_id() -> usize { + #[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))] + { + let tp: usize; + // SAFETY: reads the TCB self-pointer at fs:0, which the x86-64 Linux + // ABI guarantees is present and thread-unique. No memory is written. + unsafe { + core::arch::asm!("mov {}, fs:0", out(reg) tp, + options(nostack, preserves_flags, readonly)); + } + tp + } + #[cfg(all(target_arch = "x86_64", target_os = "windows", not(miri)))] + { + let teb: usize; + // SAFETY: gs:0x30 is the TEB self-pointer on x86-64 Windows — + // present in every thread, unique, read-only here. + unsafe { + core::arch::asm!("mov {}, gs:0x30", out(reg) teb, + options(nostack, preserves_flags, readonly)); + } + teb + } + // AArch64 splits by OS: the register holding the thread pointer is NOT the + // same one on Darwin as everywhere else. + #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple"), not(miri)))] + { + let tp: usize; + // SAFETY: on the standard AArch64 ABI (Linux, Android, BSD) tpidr_el0 + // IS the thread pointer; read-only. + unsafe { + core::arch::asm!("mrs {}, tpidr_el0", out(reg) tp, + options(nostack, preserves_flags, readonly)); + } + tp + } + #[cfg(all(target_arch = "aarch64", target_vendor = "apple", not(miri)))] + { + let tp: usize; + // SAFETY: read-only system-register read. + // + // DARWIN IS NOT THE STANDARD AArch64 ABI. Apple puts the thread pointer + // in tpidrRO_el0 and uses tpidr_el0 for the CPU/cluster id. Reading + // tpidr_el0 here — as every other AArch64 target correctly does — was a + // memory-safety bug, measured on macOS 26 / M-series: + // + // * tpidr_el0 returns small values (0x1002, 0x2005, …), not pointers; + // * it CHANGES under a single thread as that thread migrates between + // cores (5 distinct values over 3M reads on the main thread); + // * and 8 live threads produced only 5 distinct values — DIFFERENT + // THREADS COLLIDED. + // + // thread_id() is the allocator's ownership identity: `segment.thread_id` + // decides whether a free takes the local (unsynchronised) path or the + // remote one. Colliding ids let one thread free into another thread's + // segment through the owner path, racing non-atomic page state — heap + // corruption, not merely a lost optimisation. A drifting id separately + // makes a thread stop recognising its own segments. + // + // tpidrro_el0 is the real thread pointer: stable within a thread and + // unique across live threads (it is `pthread_self() + 0xe0`). Apple + // documents the LOW 3 BITS as the current CPU number, so they must be + // masked off — that is precisely the drift observed above. + unsafe { + core::arch::asm!("mrs {}, tpidrro_el0", out(reg) tp, + options(nostack, preserves_flags, readonly)); + } + tp & !0b111 + } + // Every other target: the cached-TLS path (still far cheaper than asking + // the OS on each call). + #[cfg(not(any( + all(target_arch = "x86_64", target_os = "linux", not(miri)), + all(target_arch = "x86_64", target_os = "windows", not(miri)), + all(target_arch = "aarch64", not(miri)) + )))] + { + let t = TID.with(|c| c.get()); + if t != 0 { t } else { init_tid() } + } +} + +#[cold] +#[cfg_attr( + any( + all(target_arch = "x86_64", target_os = "linux", not(miri)), + all(target_arch = "x86_64", target_os = "windows", not(miri)), + all(target_arch = "aarch64", not(miri)) + ), + allow(dead_code) +)] +fn init_tid() -> usize { + let t = prim::thread_id(); + TID.with(|c| c.set(t)); + t +} + +/// Snapshot-visit every registered heap (stats aggregation). The visitor +/// receives a COPY of the heap's counters. +/// +/// **This read is racy by design and `volatile` does not make it less so** — +/// clippy's `volatile_composites` is right to flag it, and the lint is +/// allowed HERE rather than workspace-wide so the reasoning stays attached to +/// the one site that does it. `read_volatile` on a composite gives no +/// per-field atomicity and no ordering, so a concurrent owner mutating its +/// counters can yield a snapshot whose fields disagree with each other. That +/// is accepted: these are `#[cfg(debug_assertions)]` diagnostic counters +/// (upstream's `MI_STAT` rule), nothing branches on them, and the alternative +/// — taking `&` to a struct another thread is writing — would be UB rather +/// than merely imprecise. The real fix is making the counters atomics, which +/// is the tracked follow-up (R-005 in the hardening audit). +#[allow( + clippy::volatile_composites, + reason = "racy diagnostic snapshot; see the doc comment. Tracked as R-005." +)] +pub fn for_each_heap(f: &mut dyn FnMut(&crate::heap::Heap)) { + heaps_lock(); + let mut hb = HEAPS_HEAD.load(Ordering::Acquire); + while !hb.is_null() { + // SAFETY: registry entries are live boxes. The read is unsynchronised + // with the owning thread's writes (see above) but stays within a live, + // correctly-aligned allocation for the whole of the copy. + unsafe { + let snapshot = core::ptr::read_volatile((*hb).heap.get()); + f(&snapshot); + hb = (*hb).next_box.load(Ordering::Acquire); + } + } + heaps_unlock(); +} + +/// Get (or create) the calling thread's heap box. NULL when creation fails +/// (memory exhaustion) — callers translate that into their own failure value +/// (a null allocation, a no-op collect); the TLS slot keeps the sentinel so a +/// later call retries. +#[inline] +pub fn heap_box() -> *mut HeapBox { + let hb = heap_tls::get(); + if hb == empty_heap_box_ptr() { + init_thread_heap() + } else { + hb + } +} + +/// This thread's heap box WITHOUT the created-yet test — may return the +/// shared empty sentinel. For the malloc fast path only. +/// +/// # Safety (for callers) +/// The returned box may be [`EMPTY_HEAP_BOX`], which is shared between every +/// uninitialised thread: callers must perform RAW-POINTER READS ONLY and must +/// never form a `&mut` (or write) through it. Route any miss to +/// [`ensure_heap`] before mutating anything. +#[inline] +pub fn heap_box_fast() -> *mut HeapBox { + heap_tls::get() +} + +/// Upgrade a possibly-sentinel box from [`heap_box_fast`] to this thread's +/// real, initialised heap box. NULL when creation fails (see [`heap_box`]). +#[inline] +pub fn ensure_heap(hb: *mut HeapBox) -> *mut HeapBox { + if hb == empty_heap_box_ptr() { + init_thread_heap() + } else { + hb + } +} + +/// Allocate + initialize a HeapBox (shared by thread bootstrap and +/// `mi_heap_new*`). Registered in the global registry; NOT installed in TLS. +/// +/// Returns NULL when the OS refuses the mapping (memory exhaustion). This +/// used to `expect()` — which, under the release profile's `panic = "abort"`, +/// turned OOM into a process abort on a path reachable from any fresh +/// thread's first `malloc`. The malloc contract on OOM is a null return, and +/// upstream returns NULL from `mi_heap_new` for the same reason (H-18/R-003 +/// in the hardening audit). +pub fn create_heap(tag: i32, allow_destroy: bool, arena_id: i32) -> *mut HeapBox { + // First-class heaps are not stored in `done_slot`. A thread that only + // ever called `mi_heap_new` / `heap_malloc` never ran `init_thread_heap`, + // so the TLS destructor never fired and leftover heaps stayed DELAYED + // (OH-rusty_alloc-13). Bootstrap the backing heap (and the exit hook) + // first. `init_thread_heap` calls `create_heap_uninstalled`, not this + // function, so there is no recursion. + if !crate::ONE_THREAD && heap_tls::get() == empty_heap_box_ptr() { + let _ = heap_box(); + } + create_heap_uninstalled(tag, allow_destroy, arena_id) +} + +/// Released heap boxes kept for the next heap, as upstream keeps +/// `mi_thread_data_t` (`TD_CACHE_SIZE`). A thread pool spawns and retires +/// threads constantly; without this, every thread's heap box was a fresh +/// `mmap` at its first allocation and a `munmap` at its exit, plus the +/// first-touch fault on the new mapping. A box in the cache is unregistered +/// and referenced by nothing (exactly the state in which it used to be +/// unmapped), and `create_heap_uninstalled` rewrites the whole struct. +static HEAP_BOX_CACHE: [AtomicPtr; 8] = [const { AtomicPtr::new(ptr::null_mut()) }; 8]; + +/// A cached heap box, or null. +fn heap_box_cache_take() -> *mut HeapBox { + for slot in &HEAP_BOX_CACHE { + if !slot.load(Ordering::Relaxed).is_null() { + let hb = slot.swap(ptr::null_mut(), Ordering::Acquire); + if !hb.is_null() { + return hb; + } + } + } + ptr::null_mut() +} + +/// Park a released heap box for reuse; false when the cache is full. +fn heap_box_cache_put(hb: *mut HeapBox) -> bool { + for slot in &HEAP_BOX_CACHE { + if slot + .compare_exchange(ptr::null_mut(), hb, Ordering::Release, Ordering::Relaxed) + .is_ok() + { + return true; + } + } + false +} + +fn create_heap_uninstalled(tag: i32, allow_destroy: bool, arena_id: i32) -> *mut HeapBox { + let size = core::mem::size_of::(); + // On a one-region target the FIRST descriptor is a static of the fixed + // backend's, so that a firmware's region is whole segments and needs no + // page carved out of it — the page is what made an aligned container of + // the region pad itself out to the next segment + // (`docs/plans/finished/region-alignment-bug.md` §7). Every later heap, + // and every heap on a hosted target, takes a page as before. + let first = if crate::ONE_REGION { + prim::fixed::take_first_heap_box() + } else { + None + }; + let hb: *mut HeapBox = match first { + Some(p) => p, + None => { + let cached = heap_box_cache_take(); + if cached.is_null() { + let Ok(block) = os::alloc_aligned(size, os::page_size(), true, false) else { + return ptr::null_mut(); + }; + block.ptr.cast() + } else { + cached + } + } + }; + // SAFETY: fresh committed mapping large enough for HeapBox; we initialize + // every field before publishing the pointer. + unsafe { + ptr::write( + hb, + HeapBox { + delayed: DelayedList::new(), + next_box: AtomicPtr::new(ptr::null_mut()), + owner_tid: thread_id(), + // Captured HERE, on the live creating thread, where the + // thread-local is unambiguously valid. + subproc: AtomicUsize::new(my_subproc()), + tag, + allow_destroy, + // A new heap starts as a copy of the empty one. On a + // one-region target that copy is taken from the sentinel + // itself, which already holds the whole template in flash; + // materialising `Heap::new()` here as well cost a second + // ~900-byte `.rodata` blob for the bin table. Hosted builds + // keep the constant, whose codegen was measured in Ir. + heap: UnsafeCell::new(if crate::ONE_REGION { + // SAFETY (within the enclosing block): a raw, + // non-overlapping read of the never-written sentinel's + // heap — the only access `heap_box_fast`'s contract + // permits, and `Heap` owns nothing that a bitwise copy + // could double-free. + ptr::read((*empty_heap_box_ptr()).heap.get()) + } else { + Heap::new() + }), + }, + ); + (*(*hb).heap.get()).delayed = &raw const (*hb).delayed; + (*(*hb).heap.get()).arena_id = arena_id; + (*(*hb).heap.get()).tag = tag; + // Per-heap CSPRNG stream (free-list keys, guarded sampling) -- seeded + // only where something draws from it. A build with neither `secure` + // nor guard pages has no consumer, and seeding still cost the ChaCha + // block function plus the splitmix fallback in flash. + if crate::RNG_USED { + (*(*hb).heap.get()).rng.reseed(); + } + // Guarded objects follow the option table (MI_GUARDED-equivalent) -- + // where a guard page can exist at all. `GUARD_PAGES` first, so that on + // a target without one the option reads fold away with the sampler + // (`docs/plans/finished/firmware-code-size.md`, lever 3). + let rate = crate::options::get(33).max(0) as usize; // guarded_sample_rate + let gmin = crate::options::get(30).max(0) as usize; // guarded_min + let gmax = crate::options::get(31).max(0) as usize; // guarded_max + if crate::GUARD_PAGES && gmax > 0 { + (*(*hb).heap.get()).guarded_set_size_bound(gmin, gmax); + (*(*hb).heap.get()) + .guarded_set_sample_rate(rate, crate::options::get(34).max(0) as usize); + } + } + heaps_register(hb); + hb +} + +#[cold] +fn init_thread_heap() -> *mut HeapBox { + let hb = create_heap_uninstalled(0, false, -1); + if hb.is_null() { + // OOM: install nothing — the TLS slot keeps the sentinel, so the + // NEXT allocation retries creation instead of dereferencing null. + return ptr::null_mut(); + } + heap_tls::set(hb); + BACKING_PTR.with(|c| c.set(hb)); + // The exit hook exists to abandon this heap when its thread ENDS. On a + // single-context build no thread ends, the backend's TLS never runs a + // destructor anyway (`prim::fixed::tls_new` discards it), and creating + // the slot was a compare-and-swap, a spin and a table entry paid once for + // nothing (`docs/plans/finished/firmware-code-size.md`, lever 2). + if !crate::ONE_THREAD { + done_slot().set(hb.cast::()); + } + hb +} + +ra_thread_local! { + /// The thread's original (backing) heap — what `mi_heap_get_backing` + /// returns regardless of `mi_heap_set_default` swaps. + static BACKING_PTR: Cell<*mut HeapBox> = const { Cell::new(ptr::null_mut()) }; +} + +/// `mi_heap_get_backing`. +pub fn backing_heap() -> *mut HeapBox { + let b = BACKING_PTR.with(|c| c.get()); + if b.is_null() { + let _ = heap_box(); // bootstraps and sets BACKING_PTR + BACKING_PTR.with(|c| c.get()) + } else { + b + } +} + +/// `mi_heap_set_default`: install `hb` as this thread's default; returns the +/// previous default. +/// +/// # Safety +/// `hb` must be a live heap owned by the calling thread. +pub unsafe fn set_default_heap(hb: *mut HeapBox) -> *mut HeapBox { + let prev = heap_box(); + heap_tls::set(hb); + prev +} + +/// The process-wide TLS slot whose destructor abandons a dying thread's heap. +fn done_slot() -> prim::TlsSlot { + static RAW: AtomicUsize = AtomicUsize::new(0); + static INIT: AtomicBool = AtomicBool::new(false); + let raw = RAW.load(Ordering::Acquire); + if raw != 0 { + // SAFETY: RAW holds an into_raw of a slot created below. + return unsafe { prim::TlsSlot::from_raw(raw - 1) }; + } + if INIT + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + // A panic here would be wrong twice over. An allocator must not unwind + // into its C callers (which is why the release profile is + // panic=abort), AND the winner of this race is the only thread that + // ever publishes RAW — so unwinding past the store below would strand + // every other thread in the spin loop FOREVER, turning a rare + // resource failure into a silent process-wide hang. Fail loudly and + // deterministically instead, identically in debug and release. + let Some(slot) = prim::TlsSlot::new(Some(thread_done_cb)) else { + crate::abort(); + }; + RAW.store(slot.into_raw() + 1, Ordering::Release); // +1: 0 is the unset sentinel + } + loop { + let raw = RAW.load(Ordering::Acquire); + if raw != 0 { + // SAFETY: as above. + return unsafe { prim::TlsSlot::from_raw(raw - 1) }; + } + core::hint::spin_loop(); + } +} + +#[cfg(all(windows, not(miri)))] +unsafe extern "system" fn thread_done_cb(v: *const c_void) { + if !v.is_null() { + // SAFETY: v is the HeapBox this thread stored in the done slot. + unsafe { thread_done(v.cast_mut().cast()) }; + } +} + +#[cfg(any(not(windows), miri))] +unsafe extern "C" fn thread_done_cb(v: *mut c_void) { + if !v.is_null() { + // SAFETY: v is the HeapBox this thread stored in the done slot. + unsafe { thread_done(v.cast()) }; + } +} + +/// Abandon the dying thread's heap. Teardown order is the loom-modeled one: +/// (1) every surviving page → NEVER (spins out FREEING remotes), (2) THEN +/// drain the delayed list, (3) THEN release the heap storage. +/// +/// # Safety +/// Must run on the dying thread with `hb` its live heap box, exactly once. +pub unsafe fn thread_done(hb: *mut HeapBox) { + // SAFETY: per contract `hb` is this thread's live box; `owner_tid` is a + // plain field. + let tid = unsafe { (*hb).owner_tid }; + // SAFETY: per contract we are the owner and this runs exactly once; + // sequencing per the model. + unsafe { thread_done_one(hb) }; + // Back to the SENTINEL, not null — the slot is never null, so a + // post-teardown malloc re-enters the generic path and re-initialises. + heap_tls::set(empty_heap_box_ptr()); + // First-class heaps this thread created are not stored in `done_slot` + // (`set_default_heap` does not update it). Leaving them DELAYED meant a + // second remote free after thread exit returned (OH-rusty_alloc-13). + // Abandon every remaining heap we still own so pages go NEVER. Iterative: + // a dying thread can own many first-class heaps; do not recurse. + if !crate::ONE_THREAD { + // Stall guard: a dying thread that keeps re-initialising (empty TLS + // malloc) must not loop forever. Same order as `collect_inner`'s + // `MAX_RECLAIM`. + const MAX_OWNED_HEAPS: usize = 1 << 20; + for _ in 0..MAX_OWNED_HEAPS { + let extra = take_heap_owned_by(tid); + if extra.is_null() { + break; + } + // SAFETY: extra is a live box this dying thread still owns. + unsafe { thread_done_one(extra) }; + } + } +} + +/// Abandon one heap box (NEVER + delayed drain + unregister + free the box). +/// +/// # Safety +/// `hb` is a live heap box owned by the calling thread, not used again. +unsafe fn thread_done_one(hb: *mut HeapBox) { + // SAFETY: per contract we are the owner; sequencing per the model. + unsafe { + // Read the subprocess tag from the BOX, not from `my_subproc()`. We are + // inside a platform TLS destructor and Rust's `thread_local!` storage + // may already be torn down — on aarch64-apple-darwin it reads back 0, + // which silently dropped every tagged thread's segments into the main + // subprocess. Clamped because a corrupt tag must not index out of + // bounds on the teardown path. + let sp = (*hb).subproc.load(Ordering::Acquire).min(MAX_SUBPROCS - 1); + let h = &mut *(*hb).heap.get(); + // Retire everything that is already dead (also drains delayed once). + // TEARDOWN variant: must NOT adopt orphans into a heap we are about to + // destroy — that was the 0.3.1 segfault. + h.collect_for_teardown(); + // Walk remaining segments: transition pages to NEVER, disown, publish. + let mut seg = h.segments; + h.segments = ptr::null_mut(); + while !seg.is_null() { + let next = (*seg).next; + let end = (*seg).next_free_slice as usize; + let mut idx = segment::HEADER_SLICES; + // Running pointer, not `pages[idx]`: `Page` is not a power-of-two + // size, so indexing costs a multiply plus a bounds check on every + // carved slice of every segment a dying thread owns. + let base: *mut crate::page::Page = (&raw mut (*seg).pages).cast(); + let mut slot: *mut crate::page::Page = base.add(idx); + while idx < end { + let len = ((*slot).slice_count as usize).max(1); + if (*slot).block_size > 0 { + // ONE locked read-modify-write, not two: taking the + // cross-thread chain and publishing NEVER are the same + // word, and doing them separately paid two CAS loops per + // live page of every segment a dying thread owns. + page_collect_and_set_flag(slot, XFLAG_NEVER); + (*slot).xheap.store(0, Ordering::Release); + } + idx += len; + // `wrapping_add`, not `add`: the pointer is only ever + // DEREFERENCED while `idx < end` holds, which the span-partition + // invariant makes in-bounds — but the final advance of the loop + // computes a pointer that may be one past the carved region, and + // forming that with `add` would be UB even unread. + slot = slot.wrapping_add(len); + } + if (*seg).used_pages == 0 { + let _ = segment::segment_free(seg); + h.stats.segments_freed += 1; + } else { + (*seg).thread_id.store(0, Ordering::Release); + abandoned_push(seg, sp); + } + seg = next; + } + // Huge segments: same teardown shape (collect NEVER-bound frees, + // release dead ones, abandon the rest). + let mut hseg = h.huge_segments; + h.huge_segments = ptr::null_mut(); + while !hseg.is_null() { + let next = (*hseg).next; + let pg = &raw mut (*hseg).pages[1]; + page_collect(pg); + if (*pg).used == 0 { + let _ = segment::huge_free(hseg); + h.stats.segments_freed += 1; + } else { + page_set_flag(pg, XFLAG_NEVER); + (*pg).xheap.store(0, Ordering::Release); + (*hseg).thread_id.store(0, Ordering::Release); + abandoned_push(hseg, sp); + } + hseg = next; + } + // Post-NEVER drain: blocks a FREEING remote landed while we walked. + // Their pages are abandoned now, so free_local would be wrong — but + // these blocks belong to pages we JUST abandoned; push them back on + // their pages' own lists (flag NEVER routes them there). + let mut b = (*hb).delayed.head.swap(0, Ordering::AcqRel) as *mut crate::page::Block; + while !b.is_null() { + let next = (*b).next; + let seg = segment::segment_of(b.cast()); + let pg = segment::page_of(seg, b.cast()); + crate::page::remote_free(pg, b); + b = next; + } + // Release the heap storage and clear the fast-path pointer. The + // first descriptor on a one-region target is a static, not a page: + // nothing to return (and this path is unreachable there anyway). + heaps_unregister(hb); + // A static first box is not a page and is never cached; any other + // box is cached when there is room, and unmapped otherwise. + let kept = + (crate::ONE_REGION && prim::fixed::is_first_heap_box(hb)) || heap_box_cache_put(hb); + if !kept { + let size = core::mem::size_of::(); + let blockdesc = os::OsBlock { + ptr: hb.cast(), + size: os::page_align_up(size), + is_large: false, + is_zero: false, + }; + let _ = os::free(blockdesc); + } + } +} + +/// `mi_heap_delete`: free the heap structure, MIGRATING its live pages and +/// segments into the thread's backing heap (blocks stay valid). +/// +/// # Safety +/// `hb` must be a live first-class heap owned by the calling thread, not the +/// backing heap, used never again. +pub unsafe fn heap_delete(hb: *mut HeapBox) { + let backing = backing_heap(); + if hb == backing { + return; // deleting the backing heap is a no-op (upstream-compatible) + } + // SAFETY: owner thread per contract; absorb everything into backing. + unsafe { + debug_assert_eq!((*hb).owner_tid, thread_id()); + let h = &mut *(*hb).heap.get(); + let bh = &mut *(*backing).heap.get(); + // Teardown: this heap is being deleted, so it must not adopt orphans. + h.collect_for_teardown(); + let mut seg = h.segments; + h.segments = ptr::null_mut(); + while !seg.is_null() { + let next = (*seg).next; + // `next` is read BEFORE adopting and `seg` is not touched after, so + // a release during adoption is safe here. + let _ = bh.adopt_segment(seg); // re-homes queues + xheap (tid ours) + seg = next; + } + let mut hseg = h.huge_segments; + h.huge_segments = ptr::null_mut(); + while !hseg.is_null() { + let next = (*hseg).next; + let pg = &raw mut (*hseg).pages[1]; + (*pg).xheap.store(bh.delayed as usize, Ordering::Release); + (*hseg).next = bh.huge_segments; + bh.huge_segments = hseg; + hseg = next; + } + // Fold the dying heap's counters into backing so process totals hold. + merge_stats(&mut bh.stats, &h.stats); + release_heap_box(hb); + } +} + +/// `mi_heap_destroy`: drop EVERY block of the heap at once and release its +/// memory. Falls back to delete semantics when the heap was created with +/// `allow_destroy = false`. +/// +/// # Safety +/// As [`heap_delete`], and additionally no block of this heap may be used +/// afterwards (the C contract). +pub unsafe fn heap_destroy(hb: *mut HeapBox) { + // SAFETY: owner thread per contract. + unsafe { + if !(*hb).allow_destroy { + heap_delete(hb); + return; + } + debug_assert_eq!((*hb).owner_tid, thread_id()); + let h = &mut *(*hb).heap.get(); + // Spin out any in-flight FREEING remotes before the box dies; blocks + // they queued are dying with the heap, so the delayed list is simply + // discarded. + let mut seg = h.segments; + while !seg.is_null() { + let end = (*seg).next_free_slice as usize; + let mut idx = segment::HEADER_SLICES; + // Same running-pointer walk as `thread_done`'s — the sibling site, + // checked because a defect in one slice walk is a defect in every + // slice walk. Not exercised by `mleak` (this is `heap_destroy`, not + // thread teardown), so it carries no measurement of its own; it is + // the identical transformation on identical code. + let base: *mut crate::page::Page = (&raw mut (*seg).pages).cast(); + let mut slot: *mut crate::page::Page = base.add(idx); + while idx < end { + let len = ((*slot).slice_count as usize).max(1); + if (*slot).block_size > 0 { + page_set_flag(slot, XFLAG_NEVER); + (*slot).xheap.store(0, Ordering::Release); + } + idx += len; + slot = slot.wrapping_add(len); + } + let next = (*seg).next; + let _ = segment::segment_free(seg); + seg = next; + } + let mut hseg = h.huge_segments; + while !hseg.is_null() { + let pg = &raw mut (*hseg).pages[1]; + page_set_flag(pg, XFLAG_NEVER); + (*pg).xheap.store(0, Ordering::Release); + let next = (*hseg).next; + let _ = segment::huge_free(hseg); + hseg = next; + } + release_heap_box(hb); + } +} + +fn merge_stats(into: &mut crate::heap::Stats, from: &crate::heap::Stats) { + into.allocs += from.allocs; + into.frees += from.frees; + into.generic += from.generic; + into.pages_fresh += from.pages_fresh; + into.segments += from.segments; + into.huge_allocs += from.huge_allocs; + into.extends += from.extends; + into.large_allocs += from.large_allocs; + into.pages_retired += from.pages_retired; + into.segments_freed += from.segments_freed; + into.realloc_in_place += from.realloc_in_place; + into.realloc_moved += from.realloc_moved; + into.delayed_frees += from.delayed_frees; + into.reclaims += from.reclaims; +} + +/// Unregister and release a HeapBox, repairing the TLS default if needed. +/// +/// # Safety +/// `hb` live, owned by the calling thread, all contents already migrated or +/// released. +unsafe fn release_heap_box(hb: *mut HeapBox) { + heaps_unregister(hb); + if heap_tls::get() == hb { + heap_tls::set(backing_heap()); + } + // A first-class heap created before the first allocation would have been + // handed the fixed backend's static descriptor; that is not a page and + // must not be returned to the region. + if crate::ONE_REGION && prim::fixed::is_first_heap_box(hb) { + return; + } + if heap_box_cache_put(hb) { + return; + } + let size = core::mem::size_of::(); + let blockdesc = os::OsBlock { + ptr: hb.cast(), + size: os::page_align_up(size), + is_large: false, + is_zero: false, + }; + // SAFETY: hb's mapping came from create_heap's os::alloc_aligned with this + // exact size; unregistered above, no references remain. + unsafe { + let _ = os::free(blockdesc); + } +} + +// --------------------------------------------------------------------------- +// Global abandoned-segment list (spin-locked; cold path). +// --------------------------------------------------------------------------- + +/// Subprocess isolation (M6, plan §5.9): threads tagged with a subproc id +/// abandon/reclaim only within it — separate interpreters never exchange +/// segments. Id 0 is the main subprocess. +const MAX_SUBPROCS: usize = 64; + +static ABANDONED_LOCK: AtomicBool = AtomicBool::new(false); +static ABANDONED_HEADS: [AtomicPtr; MAX_SUBPROCS] = + [const { AtomicPtr::new(ptr::null_mut()) }; MAX_SUBPROCS]; +static SUBPROC_NEXT: AtomicUsize = AtomicUsize::new(1); +/// Diagnostic: segments currently abandoned (all subprocs). +pub static ABANDONED_COUNT: AtomicUsize = AtomicUsize::new(0); + +ra_thread_local! { + static SUBPROC: Cell = const { Cell::new(0) }; +} + +/// `mi_subproc_main` → 0. +pub fn subproc_main() -> usize { + 0 +} + +/// `mi_subproc_new`: allocate a fresh subprocess id. +/// +/// Exhaustion is an invalid id (`MAX_SUBPROCS`), not an assert (OH-rusty_alloc-21). +pub fn subproc_new() -> usize { + loop { + let id = SUBPROC_NEXT.load(Ordering::Acquire); + if id >= MAX_SUBPROCS { + return MAX_SUBPROCS; + } + if SUBPROC_NEXT + .compare_exchange_weak(id, id + 1, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return id; + } + } +} + +/// `mi_subproc_delete`: ids are not recycled in v1; abandoned segments of the +/// subproc are drained into the main subproc so they stay reclaimable. +pub fn subproc_delete(id: usize) { + if id == 0 || id >= MAX_SUBPROCS { + return; + } + abandoned_lock(); + // SAFETY: list surgery under the lock. + unsafe { + let mut seg = ABANDONED_HEADS[id].swap(ptr::null_mut(), Ordering::AcqRel); + while !seg.is_null() { + let next = (*seg).next; + (*seg).next = ABANDONED_HEADS[0].load(Ordering::Relaxed); + ABANDONED_HEADS[0].store(seg, Ordering::Relaxed); + seg = next; + } + } + abandoned_unlock(); +} + +/// `mi_subproc_add_current_thread`. +pub fn subproc_add_current_thread(id: usize) { + if id >= MAX_SUBPROCS { + return; + } + SUBPROC.with(|c| c.set(id)); + // Mirror onto this thread's backing heap so TEARDOWN can read the tag + // without touching the thread-local (see `HeapBox::subproc`). Tagging + // before the first allocation is the common order and is covered by + // `create_heap` reading `my_subproc()`; this covers tagging AFTER the heap + // already exists. Deliberately does NOT bootstrap a heap — a thread that + // tags itself and never allocates should not be given one. + let hb = BACKING_PTR.with(|c| c.get()); + if !hb.is_null() { + // SAFETY: hb is this thread's own live backing box; `subproc` is + // atomic, so the teardown read needs no other synchronisation. + unsafe { (*hb).subproc.store(id, Ordering::Release) }; + } +} + +fn my_subproc() -> usize { + SUBPROC.with(|c| c.get()) +} + +/// `mi_abandoned_visit_blocks`: visit live pages/blocks of a subprocess's +/// abandoned segments, holding the list lock (segments are pinned meanwhile). +pub fn abandoned_visit_blocks( + subproc_id: usize, + heap_tag: i32, + visit_blocks: bool, + f: &mut dyn FnMut(&crate::heap::AreaInfo, *mut u8, usize) -> bool, +) -> bool { + if subproc_id >= MAX_SUBPROCS { + return false; + } + abandoned_lock(); + let mut ok = true; + let mut seg = ABANDONED_HEADS[subproc_id].load(Ordering::Acquire); + while !seg.is_null() && ok { + // SAFETY: the lock pins abandoned segments (adopt also takes it). + unsafe { + ok = crate::heap::visit_segment_blocks(seg, heap_tag, visit_blocks, false, f); + seg = (*seg).next; + } + } + abandoned_unlock(); + ok +} + +fn abandoned_lock() { + while ABANDONED_LOCK + .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + core::hint::spin_loop(); + } +} + +fn abandoned_unlock() { + ABANDONED_LOCK.store(false, Ordering::Release); +} + +/// Publish `seg` to subprocess `sp`'s abandoned list. +/// +/// `sp` is passed in rather than read from `my_subproc()`: every caller is on +/// the teardown path, where the `SUBPROC` thread-local may already be gone (see +/// `HeapBox::subproc`). The owning heap box carries the tag instead. +fn abandoned_push(seg: *mut Segment, sp: usize) { + // The correctness argument behind `abandoned_pop`'s early return: if this + // ever runs on a single-context build, a thread ended, and the target's + // single-thread assertion was false all along. Worth knowing for its own + // sake -- `SingleThreadCell` and `prim::fixed` rest on the same claim. + if crate::ONE_THREAD { + unreachable!("a segment was abandoned on a build that asserted a single thread"); + } + debug_assert!(sp < MAX_SUBPROCS); + // PURGE BEFORE ORPHANING (`abandoned_page_purge`, on by default). Until + // some other thread adopts this segment it has no owner, so every page it + // touched stays resident for an unbounded time and nothing can reuse it. + // Done here, before publishing to the list, because this is the last + // instant the caller still exclusively owns the segment. + if crate::options::is_enabled(30) { + // SAFETY: still owned by the dying thread; free spans hold no live + // blocks, and each purged span is marked so reuse re-commits it. + unsafe { crate::segment::purge_free_spans(seg) }; + } + abandoned_lock(); + // SAFETY: seg is disowned; its `next` is ours to use as the list link. + unsafe { + (*seg).next = ABANDONED_HEADS[sp].load(Ordering::Relaxed); + } + ABANDONED_HEADS[sp].store(seg, Ordering::Relaxed); + ABANDONED_COUNT.fetch_add(1, Ordering::Relaxed); + abandoned_unlock(); +} + +/// Pop one abandoned segment from the CALLER's subprocess and take ownership. +/// Returns null when none are available. +pub fn abandoned_pop() -> *mut Segment { + // A single-context build has nothing to pop, by construction: a segment is + // only ever abandoned by a thread that ENDS (`thread_done`), and the + // target has asserted there is exactly one thread for the life of the + // program. Folding this to null is what makes `adopt_segment` unreachable + // from the allocation path, so the linker drops it -- 1,154 bytes on an + // ESP32-S3 that could never have run (`docs/plans/finished/firmware-code-size.md`, + // lever 1). `heap_delete` still calls it directly, and a firmware that + // deletes first-class heaps keeps it; that is a real path, not a + // cross-thread one. + if crate::ONE_THREAD { + return ptr::null_mut(); + } + let sp = my_subproc(); + if ABANDONED_HEADS[sp].load(Ordering::Acquire).is_null() { + return ptr::null_mut(); // fast no-lock exit for the common case + } + abandoned_lock(); + let seg = ABANDONED_HEADS[sp].load(Ordering::Relaxed); + if !seg.is_null() { + // SAFETY: list links are protected by the lock. + unsafe { + ABANDONED_HEADS[sp].store((*seg).next, Ordering::Relaxed); + (*seg).next = ptr::null_mut(); + (*seg).thread_id.store(thread_id(), Ordering::Release); + } + ABANDONED_COUNT.fetch_sub(1, Ordering::Relaxed); + } + abandoned_unlock(); + seg +}