From 7f55a7d7423a47481749dd595f1f0dab2690e060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 00:22:52 +0200 Subject: [PATCH] perf(gc): batch dead old-object page unregistration per sweep step A full collection unregistered each dead old object from the page index on its own: two `Vec` allocations, a deferral-buffer flush, a promoted-run materialization, both table borrows, and a linear `position` over the page's object list before `swap_remove`. Freeing every object on a page that way is quadratic in objects per page, and a full frees whole pages of small objects. On records_array_8m:scan (~720k dead records per full) `invalidate_dead_old_arena_header` was 15.8% of ALL samples -- the single largest cost of a full collection. The registration side already batches for exactly this reason (`flush_deferred_old_page_registrations_batch`, #7624). This is its mirror: `ArenaSweepObjectsState` still invalidates each dead header's fields immediately, so no walker can read it as live, but queues the page-index removal and flushes once per sweep step (and every 4096 headers, so an unbudgeted sweep never stages an unbounded buffer). The batch does one flush, one run materialization per touched page, one `retain` per page against that page's sorted dead headers, and one page-meta update per page -- allocated bytes and object counts only fall here, so applying a page's decrements together and resetting/refreshing once is the same state the per-object path reaches. Nothing inside a sweep step reads page-index membership, and the queue is empty at every step boundary. Four fulls on records_array_8m:scan: 241 ms -> 187 ms (-22%). The batched flush is 6.3% of samples where the per-object remover was 15.8%. `every_page_object_reader_expands_promoted_runs` covers the new remover (it touches OLD_GEN_PAGE_OBJECTS and expands first). A new test drives both removers over the same population in one arena -- page-spanning objects, a fully emptied page, partial pages -- and requires identical page membership and metadata; it was sabotage-tested (dropping the object-count decrement fails it with "page metadata diverged"). --- crates/perry-runtime/src/arena/mod.rs | 6 +- .../perry-runtime/src/arena/page_meta/mod.rs | 109 ++++++++++++++++ crates/perry-runtime/src/arena/tests.rs | 2 +- .../src/arena/tests_batch_unregister.rs | 123 ++++++++++++++++++ crates/perry-runtime/src/gc/oldgen.rs | 16 ++- .../src/gc/oldgen/sweep_batch.rs | 46 +++++++ 6 files changed, 295 insertions(+), 7 deletions(-) create mode 100644 crates/perry-runtime/src/arena/tests_batch_unregister.rs create mode 100644 crates/perry-runtime/src/gc/oldgen/sweep_batch.rs diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index ecf81353a9..ad3b0bf4e1 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -28,6 +28,8 @@ mod walk; #[cfg(test)] mod tests; #[cfg(test)] +mod tests_batch_unregister; +#[cfg(test)] mod tests_promoted_runs; // Cross-sibling shared types/thread-locals (used by sibling modules via @@ -152,8 +154,8 @@ pub(crate) use page_meta::{ old_page_account_swept_object, old_page_clear_dirty, old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle, old_pages_reset_sweep_accounting, record_arena_object_start, unregister_old_object_pages, - HeapGeneration, HeapSpace, OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, - OldPageSummary, + unregister_old_objects_batch, HeapGeneration, HeapSpace, OldArenaPageObjectCursor, + OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, }; #[cfg(test)] diff --git a/crates/perry-runtime/src/arena/page_meta/mod.rs b/crates/perry-runtime/src/arena/page_meta/mod.rs index 708b16b90f..f2f0b180b6 100644 --- a/crates/perry-runtime/src/arena/page_meta/mod.rs +++ b/crates/perry-runtime/src/arena/page_meta/mod.rs @@ -1285,6 +1285,115 @@ pub(crate) fn unregister_old_object_pages(header_addr: usize, total_size: usize) update_old_page_meta_for_object(&removed_pages, false); } +/// Batched [`unregister_old_object_pages`] for the dead old objects one sweep +/// step found. +/// +/// The per-object remover allocates two `Vec`s, flushes the deferral buffer, +/// borrows both tables, and then finds the header in its page's object list +/// with a linear `position` before `swap_remove`. Freeing every object on a +/// page that way is quadratic in objects per page, and a full collection frees +/// whole pages of small objects: on records_array_8m:scan (~720k dead records +/// per full) it was 15.8% of ALL samples, the single largest cost of a full +/// collection. The registration side already batches for the same reason +/// (`flush_deferred_old_page_registrations_batch`); this is its mirror. +/// +/// One flush, one run materialization per touched page, then one `retain` per +/// page against that page's dead headers (sorted, so membership is a binary +/// search). Page-meta decrements for a page are applied together and its +/// reset/refresh run once: allocated bytes and object counts only fall here, +/// so "both reached zero at some point" and "both are zero at the end" are the +/// same event, and `reset_cycle_sweep_accounting` / `refresh_policy_bits` are +/// pure recomputes of the page's own fields. +/// +/// `scratch` is caller-owned and reused across steps, so a flush allocates +/// nothing once warm (the #7624 lesson: re-growing staging buffers per batch +/// cost +31 MB peak RSS on json_pipeline). +pub(crate) fn unregister_old_objects_batch( + dead: &[(usize, usize)], + scratch: &mut Vec<(usize, usize, usize)>, +) { + if dead.is_empty() { + return; + } + flush_deferred_old_page_registrations(); + scratch.clear(); + for &(header_addr, total_size) in dead { + if header_addr == 0 || total_size == 0 { + continue; + } + let object_end = header_addr + total_size; + let first_page = generation_page_for_addr(header_addr); + let last_page = generation_page_for_addr(object_end - 1); + for page in first_page..=last_page { + let page_base = generation_page_base(page); + let overlap_start = header_addr.max(page_base); + let overlap_end = object_end.min(page_base + GENERATION_PAGE_SIZE); + if overlap_start < overlap_end { + scratch.push((page, header_addr, overlap_end - overlap_start)); + } + } + } + // RUN REMOVER, as the per-object path: expand before touching membership. + // Done before borrowing the index, because expansion writes it. + if OLD_GEN_PAGE_PROMOTED_RUNS_NONEMPTY.with(Cell::get) { + let mut last = usize::MAX; + for &(page, _, _) in scratch.iter() { + if page != last { + materialize_promoted_page_runs(core::iter::once(page)); + last = page; + } + } + } + scratch.sort_unstable_by_key(|&(page, header, _)| (page, header)); + OLD_GEN_PAGE_OBJECTS.with(|index| { + OLD_GEN_PAGE_META.with(|meta| { + let mut index = index.borrow_mut(); + let mut meta = meta.borrow_mut(); + let mut start = 0; + while start < scratch.len() { + let page = scratch[start].0; + let mut end = start; + while end < scratch.len() && scratch[end].0 == page { + end += 1; + } + let group = &scratch[start..end]; + let mut removed_bytes = 0usize; + let mut removed_objects = 0usize; + let mut remove_page = false; + if let Some(headers) = index.get_mut(&page) { + headers.retain(|&addr| { + match group.binary_search_by_key(&addr, |&(_, header, _)| header) { + Ok(i) => { + removed_bytes = removed_bytes.saturating_add(group[i].2); + removed_objects += 1; + false + } + Err(_) => true, + } + }); + remove_page = headers.is_empty(); + } + if remove_page { + index.remove(&page); + } + if removed_objects != 0 { + let page_meta = meta + .entry(page) + .or_insert_with(|| OldPageMeta::zero_for_page(page)); + page_meta.allocated_bytes = + page_meta.allocated_bytes.saturating_sub(removed_bytes); + page_meta.object_count = page_meta.object_count.saturating_sub(removed_objects); + if page_meta.allocated_bytes == 0 && page_meta.object_count == 0 { + page_meta.reset_cycle_sweep_accounting(); + } + page_meta.refresh_policy_bits(); + } + start = end; + } + }); + }); +} + pub(crate) fn old_pages_begin_gc_cycle() { // #7624 CYCLE START: all three cycle constructors route through here // (`gc/mod.rs`'s minor, `gc/cycle.rs`'s `new_full`, `gc/policy.rs`'s diff --git a/crates/perry-runtime/src/arena/tests.rs b/crates/perry-runtime/src/arena/tests.rs index 9dadc3a0a0..ab319b8053 100644 --- a/crates/perry-runtime/src/arena/tests.rs +++ b/crates/perry-runtime/src/arena/tests.rs @@ -211,7 +211,7 @@ fn old_page_meta(page: usize) -> OldPageMeta { old_page_meta_for_tests(page).expect("old page metadata should be registered") } -fn old_header_and_size(user_ptr: usize) -> (usize, usize) { +pub(super) fn old_header_and_size(user_ptr: usize) -> (usize, usize) { let header_addr = user_ptr - GC_HEADER_SIZE; let total_size = unsafe { (*(header_addr as *const GcHeader)).size as usize }; (header_addr, total_size) diff --git a/crates/perry-runtime/src/arena/tests_batch_unregister.rs b/crates/perry-runtime/src/arena/tests_batch_unregister.rs new file mode 100644 index 0000000000..c21ca33a73 --- /dev/null +++ b/crates/perry-runtime/src/arena/tests_batch_unregister.rs @@ -0,0 +1,123 @@ +//! The batched old-object page unregister must be indistinguishable from the +//! per-object remover it replaces on the full sweep's hot path. +//! +//! Both are driven over the SAME population in one fresh arena: remove a subset +//! one object at a time and snapshot, restore the subset, remove it again in +//! one batch and snapshot, then require the two snapshots to be identical -- +//! page membership as walked, and every touched page's metadata. The subset +//! mixes page-spanning objects, fully emptied pages and partial pages, because +//! those are the three shapes the grouping and the per-page reset/refresh +//! have to get right. + +use super::page_meta::{ + old_arena_walk_objects_on_pages, old_object_page_overlaps, old_page_meta_for_tests, + register_old_object_pages, unregister_old_object_pages, unregister_old_objects_batch, + OldPageMeta, GENERATION_PAGE_SIZE, +}; +use super::tests::old_header_and_size as header_and_size; +use super::*; +use crate::gc::GC_TYPE_STRING; + +type Snapshot = (Vec, Vec<(usize, Option)>); + +fn snapshot(pages: &[usize]) -> Snapshot { + let mut set = crate::fast_hash::new_ptr_hash_set(); + for &page in pages { + set.insert(page); + } + let mut members = Vec::new(); + old_arena_walk_objects_on_pages(&set, |header| members.push(header as usize)); + members.sort_unstable(); + let metas = pages + .iter() + .map(|&page| (page, old_page_meta_for_tests(page))) + .collect(); + (members, metas) +} + +#[test] +fn batched_unregister_leaves_the_same_index_and_metadata_as_per_object_removal() { + super::tests::run_with_fresh_arenas(|| { + // Many small objects per page, plus objects wider than a page. + let mut objects = Vec::new(); + for i in 0..600 { + let size = if i % 97 == 0 { + GENERATION_PAGE_SIZE + 512 + } else { + 72 + }; + let ptr = arena_alloc_gc_old(size, 8, GC_TYPE_STRING) as usize; + objects.push(header_and_size(ptr)); + } + let mut pages: Vec = objects + .iter() + .flat_map(|&(h, s)| old_object_page_overlaps(h, s).into_iter().map(|(p, _)| p)) + .collect(); + pages.sort_unstable(); + pages.dedup(); + + // Every third object, every page-spanning object, and EVERY object on + // one chosen page, so at least one page empties completely. + let emptied_page = pages[pages.len() / 2]; + let subset: Vec<(usize, usize)> = objects + .iter() + .enumerate() + .filter(|&(i, &(h, s))| { + i % 3 == 0 + || s > GENERATION_PAGE_SIZE + || old_object_page_overlaps(h, s) + .iter() + .any(|&(p, _)| p == emptied_page) + }) + .map(|(_, &o)| o) + .collect(); + assert!(subset.len() > 200 && subset.len() < objects.len()); + + let before = snapshot(&pages); + + for &(h, s) in &subset { + unregister_old_object_pages(h, s); + } + let per_object = snapshot(&pages); + assert_ne!(per_object, before, "the subset must actually be removed"); + + for &(h, s) in &subset { + register_old_object_pages(h, s); + } + assert_eq!( + snapshot(&pages).0, + before.0, + "restoring the subset must restore page membership" + ); + + let mut scratch = Vec::new(); + unregister_old_objects_batch(&subset, &mut scratch); + let batched = snapshot(&pages); + + assert_eq!( + batched.0, per_object.0, + "batched removal must leave exactly the headers per-object removal leaves" + ); + for ((page, a), (_, b)) in per_object.1.iter().zip(batched.1.iter()) { + let fields = |m: &Option| { + m.map(|m| { + ( + m.allocated_bytes, + m.object_count, + m.live_bytes, + m.dead_bytes, + m.live_object_count, + m.dead_object_count, + m.evacuation_eligible, + ) + }) + }; + assert_eq!(fields(a), fields(b), "page {page:#x} metadata diverged"); + } + assert!( + per_object.1.iter().any(|(p, m)| *p == emptied_page + && m.map_or(true, |m| m.object_count == 0 && m.allocated_bytes == 0)), + "the chosen page must be emptied, or the reset path went unexercised" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/oldgen.rs b/crates/perry-runtime/src/gc/oldgen.rs index 0c0ef47b5d..0a96aea979 100644 --- a/crates/perry-runtime/src/gc/oldgen.rs +++ b/crates/perry-runtime/src/gc/oldgen.rs @@ -1345,6 +1345,8 @@ impl IncrementalSweepState { struct ArenaSweepObjectsState { cursor: crate::arena::ArenaObjectCursor, + /// Dead old headers awaiting one batched page-index removal (see `sweep_batch`). + pending_old_unregister: sweep_batch::PendingOldUnregister, block_snapshots: Vec, block_has_live: Vec, resettable_general_n: usize, @@ -1404,6 +1406,7 @@ impl ArenaSweepObjectsState { crate::arena::old_pages_reset_sweep_accounting(); Self { cursor: crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::BlockIndex), + pending_old_unregister: Default::default(), block_snapshots, block_has_live: vec![false; n_blocks], resettable_general_n: crate::arena::general_block_count(), @@ -1445,14 +1448,18 @@ impl ArenaSweepObjectsState { fn step(&mut self, budget: usize) -> bool { let mut remaining = budget; + let mut done = false; while remaining > 0 { let Some((header_ptr, block_idx)) = self.cursor.next() else { - return true; + done = true; + break; }; remaining -= 1; self.process_object(header_ptr as *mut GcHeader, block_idx); } - false + // Never leave a dead header in the page index across a step boundary. + self.pending_old_unregister.flush(); + done } fn block_has_live(&self) -> &[bool] { @@ -1624,7 +1631,7 @@ impl ArenaSweepObjectsState { gc_type_clear_dead_payload_side_tables((*header).obj_type, user_ptr as usize); } if self.reclaim_dead_old_blocks && dead_old { - invalidate_dead_old_arena_header(header, total_size); + self.pending_old_unregister.defer(header, total_size); } else { (*header).gc_flags = flags & !(GC_FLAG_FORWARDED | GC_FLAG_MARKED); } @@ -1643,7 +1650,7 @@ impl ArenaSweepObjectsState { } finalize_dead_arena_payload(header, user_ptr, self.overflow_active); if self.reclaim_dead_old_blocks && dead_old { - invalidate_dead_old_arena_header(header, total_size); + self.pending_old_unregister.defer(header, total_size); } } } @@ -1656,6 +1663,7 @@ enum ArenaSweepCleanupSubphase { Done, } +mod sweep_batch; mod sweep_cleanup; use sweep_cleanup::*; diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs new file mode 100644 index 0000000000..db13089b1c --- /dev/null +++ b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs @@ -0,0 +1,46 @@ +//! Batched page-index removal for the dead old objects a sweep step frees. +//! +//! `invalidate_dead_old_arena_header` unregisters each dead old object from the +//! page index on its own, and that removal is quadratic in objects per page (see +//! `arena::unregister_old_objects_batch`). A full collection frees whole pages +//! of small objects, so the sweep queues them here and removes a step's worth in +//! one pass. The header fields are still invalidated immediately, exactly as +//! before, so no walker can read a dead header as a live object in between. + +use super::super::*; + +/// Flush once this many dead headers are queued, so a single unbudgeted sweep +/// of a large heap never stages an unbounded buffer. +const FLUSH_AT: usize = 4096; + +#[derive(Default)] +pub(super) struct PendingOldUnregister { + dead: Vec<(usize, usize)>, + scratch: Vec<(usize, usize, usize)>, +} + +impl PendingOldUnregister { + /// Invalidate a dead old header now and queue its page-index removal. + /// + /// # Safety + /// + /// `header` must be a dead old-gen arena header of `total_size` bytes. + pub(super) unsafe fn defer(&mut self, header: *mut GcHeader, total_size: usize) { + (*header).obj_type = 0; + (*header).gc_flags = 0; + (*header)._reserved = 0; + self.dead.push((header as usize, total_size)); + if self.dead.len() >= FLUSH_AT { + self.flush(); + } + } + + /// Remove every queued header from the page index. + pub(super) fn flush(&mut self) { + if self.dead.is_empty() { + return; + } + crate::arena::unregister_old_objects_batch(&self.dead, &mut self.scratch); + self.dead.clear(); + } +}