diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index ecf81353a9..ad3b0bf4e1 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -28,6 +28,8 @@ mod walk; #[cfg(test)] mod tests; #[cfg(test)] +mod tests_batch_unregister; +#[cfg(test)] mod tests_promoted_runs; // Cross-sibling shared types/thread-locals (used by sibling modules via @@ -152,8 +154,8 @@ pub(crate) use page_meta::{ old_page_account_swept_object, old_page_clear_dirty, old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle, old_pages_reset_sweep_accounting, record_arena_object_start, unregister_old_object_pages, - HeapGeneration, HeapSpace, OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, - OldPageSummary, + unregister_old_objects_batch, HeapGeneration, HeapSpace, OldArenaPageObjectCursor, + OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, }; #[cfg(test)] diff --git a/crates/perry-runtime/src/arena/page_meta/mod.rs b/crates/perry-runtime/src/arena/page_meta/mod.rs index 708b16b90f..f2f0b180b6 100644 --- a/crates/perry-runtime/src/arena/page_meta/mod.rs +++ b/crates/perry-runtime/src/arena/page_meta/mod.rs @@ -1285,6 +1285,115 @@ pub(crate) fn unregister_old_object_pages(header_addr: usize, total_size: usize) update_old_page_meta_for_object(&removed_pages, false); } +/// Batched [`unregister_old_object_pages`] for the dead old objects one sweep +/// step found. +/// +/// The per-object remover allocates two `Vec`s, flushes the deferral buffer, +/// borrows both tables, and then finds the header in its page's object list +/// with a linear `position` before `swap_remove`. Freeing every object on a +/// page that way is quadratic in objects per page, and a full collection frees +/// whole pages of small objects: on records_array_8m:scan (~720k dead records +/// per full) it was 15.8% of ALL samples, the single largest cost of a full +/// collection. The registration side already batches for the same reason +/// (`flush_deferred_old_page_registrations_batch`); this is its mirror. +/// +/// One flush, one run materialization per touched page, then one `retain` per +/// page against that page's dead headers (sorted, so membership is a binary +/// search). Page-meta decrements for a page are applied together and its +/// reset/refresh run once: allocated bytes and object counts only fall here, +/// so "both reached zero at some point" and "both are zero at the end" are the +/// same event, and `reset_cycle_sweep_accounting` / `refresh_policy_bits` are +/// pure recomputes of the page's own fields. +/// +/// `scratch` is caller-owned and reused across steps, so a flush allocates +/// nothing once warm (the #7624 lesson: re-growing staging buffers per batch +/// cost +31 MB peak RSS on json_pipeline). +pub(crate) fn unregister_old_objects_batch( + dead: &[(usize, usize)], + scratch: &mut Vec<(usize, usize, usize)>, +) { + if dead.is_empty() { + return; + } + flush_deferred_old_page_registrations(); + scratch.clear(); + for &(header_addr, total_size) in dead { + if header_addr == 0 || total_size == 0 { + continue; + } + let object_end = header_addr + total_size; + let first_page = generation_page_for_addr(header_addr); + let last_page = generation_page_for_addr(object_end - 1); + for page in first_page..=last_page { + let page_base = generation_page_base(page); + let overlap_start = header_addr.max(page_base); + let overlap_end = object_end.min(page_base + GENERATION_PAGE_SIZE); + if overlap_start < overlap_end { + scratch.push((page, header_addr, overlap_end - overlap_start)); + } + } + } + // RUN REMOVER, as the per-object path: expand before touching membership. + // Done before borrowing the index, because expansion writes it. + if OLD_GEN_PAGE_PROMOTED_RUNS_NONEMPTY.with(Cell::get) { + let mut last = usize::MAX; + for &(page, _, _) in scratch.iter() { + if page != last { + materialize_promoted_page_runs(core::iter::once(page)); + last = page; + } + } + } + scratch.sort_unstable_by_key(|&(page, header, _)| (page, header)); + OLD_GEN_PAGE_OBJECTS.with(|index| { + OLD_GEN_PAGE_META.with(|meta| { + let mut index = index.borrow_mut(); + let mut meta = meta.borrow_mut(); + let mut start = 0; + while start < scratch.len() { + let page = scratch[start].0; + let mut end = start; + while end < scratch.len() && scratch[end].0 == page { + end += 1; + } + let group = &scratch[start..end]; + let mut removed_bytes = 0usize; + let mut removed_objects = 0usize; + let mut remove_page = false; + if let Some(headers) = index.get_mut(&page) { + headers.retain(|&addr| { + match group.binary_search_by_key(&addr, |&(_, header, _)| header) { + Ok(i) => { + removed_bytes = removed_bytes.saturating_add(group[i].2); + removed_objects += 1; + false + } + Err(_) => true, + } + }); + remove_page = headers.is_empty(); + } + if remove_page { + index.remove(&page); + } + if removed_objects != 0 { + let page_meta = meta + .entry(page) + .or_insert_with(|| OldPageMeta::zero_for_page(page)); + page_meta.allocated_bytes = + page_meta.allocated_bytes.saturating_sub(removed_bytes); + page_meta.object_count = page_meta.object_count.saturating_sub(removed_objects); + if page_meta.allocated_bytes == 0 && page_meta.object_count == 0 { + page_meta.reset_cycle_sweep_accounting(); + } + page_meta.refresh_policy_bits(); + } + start = end; + } + }); + }); +} + pub(crate) fn old_pages_begin_gc_cycle() { // #7624 CYCLE START: all three cycle constructors route through here // (`gc/mod.rs`'s minor, `gc/cycle.rs`'s `new_full`, `gc/policy.rs`'s diff --git a/crates/perry-runtime/src/arena/tests.rs b/crates/perry-runtime/src/arena/tests.rs index 9dadc3a0a0..ab319b8053 100644 --- a/crates/perry-runtime/src/arena/tests.rs +++ b/crates/perry-runtime/src/arena/tests.rs @@ -211,7 +211,7 @@ fn old_page_meta(page: usize) -> OldPageMeta { old_page_meta_for_tests(page).expect("old page metadata should be registered") } -fn old_header_and_size(user_ptr: usize) -> (usize, usize) { +pub(super) fn old_header_and_size(user_ptr: usize) -> (usize, usize) { let header_addr = user_ptr - GC_HEADER_SIZE; let total_size = unsafe { (*(header_addr as *const GcHeader)).size as usize }; (header_addr, total_size) diff --git a/crates/perry-runtime/src/arena/tests_batch_unregister.rs b/crates/perry-runtime/src/arena/tests_batch_unregister.rs new file mode 100644 index 0000000000..c21ca33a73 --- /dev/null +++ b/crates/perry-runtime/src/arena/tests_batch_unregister.rs @@ -0,0 +1,123 @@ +//! The batched old-object page unregister must be indistinguishable from the +//! per-object remover it replaces on the full sweep's hot path. +//! +//! Both are driven over the SAME population in one fresh arena: remove a subset +//! one object at a time and snapshot, restore the subset, remove it again in +//! one batch and snapshot, then require the two snapshots to be identical -- +//! page membership as walked, and every touched page's metadata. The subset +//! mixes page-spanning objects, fully emptied pages and partial pages, because +//! those are the three shapes the grouping and the per-page reset/refresh +//! have to get right. + +use super::page_meta::{ + old_arena_walk_objects_on_pages, old_object_page_overlaps, old_page_meta_for_tests, + register_old_object_pages, unregister_old_object_pages, unregister_old_objects_batch, + OldPageMeta, GENERATION_PAGE_SIZE, +}; +use super::tests::old_header_and_size as header_and_size; +use super::*; +use crate::gc::GC_TYPE_STRING; + +type Snapshot = (Vec, Vec<(usize, Option)>); + +fn snapshot(pages: &[usize]) -> Snapshot { + let mut set = crate::fast_hash::new_ptr_hash_set(); + for &page in pages { + set.insert(page); + } + let mut members = Vec::new(); + old_arena_walk_objects_on_pages(&set, |header| members.push(header as usize)); + members.sort_unstable(); + let metas = pages + .iter() + .map(|&page| (page, old_page_meta_for_tests(page))) + .collect(); + (members, metas) +} + +#[test] +fn batched_unregister_leaves_the_same_index_and_metadata_as_per_object_removal() { + super::tests::run_with_fresh_arenas(|| { + // Many small objects per page, plus objects wider than a page. + let mut objects = Vec::new(); + for i in 0..600 { + let size = if i % 97 == 0 { + GENERATION_PAGE_SIZE + 512 + } else { + 72 + }; + let ptr = arena_alloc_gc_old(size, 8, GC_TYPE_STRING) as usize; + objects.push(header_and_size(ptr)); + } + let mut pages: Vec = objects + .iter() + .flat_map(|&(h, s)| old_object_page_overlaps(h, s).into_iter().map(|(p, _)| p)) + .collect(); + pages.sort_unstable(); + pages.dedup(); + + // Every third object, every page-spanning object, and EVERY object on + // one chosen page, so at least one page empties completely. + let emptied_page = pages[pages.len() / 2]; + let subset: Vec<(usize, usize)> = objects + .iter() + .enumerate() + .filter(|&(i, &(h, s))| { + i % 3 == 0 + || s > GENERATION_PAGE_SIZE + || old_object_page_overlaps(h, s) + .iter() + .any(|&(p, _)| p == emptied_page) + }) + .map(|(_, &o)| o) + .collect(); + assert!(subset.len() > 200 && subset.len() < objects.len()); + + let before = snapshot(&pages); + + for &(h, s) in &subset { + unregister_old_object_pages(h, s); + } + let per_object = snapshot(&pages); + assert_ne!(per_object, before, "the subset must actually be removed"); + + for &(h, s) in &subset { + register_old_object_pages(h, s); + } + assert_eq!( + snapshot(&pages).0, + before.0, + "restoring the subset must restore page membership" + ); + + let mut scratch = Vec::new(); + unregister_old_objects_batch(&subset, &mut scratch); + let batched = snapshot(&pages); + + assert_eq!( + batched.0, per_object.0, + "batched removal must leave exactly the headers per-object removal leaves" + ); + for ((page, a), (_, b)) in per_object.1.iter().zip(batched.1.iter()) { + let fields = |m: &Option| { + m.map(|m| { + ( + m.allocated_bytes, + m.object_count, + m.live_bytes, + m.dead_bytes, + m.live_object_count, + m.dead_object_count, + m.evacuation_eligible, + ) + }) + }; + assert_eq!(fields(a), fields(b), "page {page:#x} metadata diverged"); + } + assert!( + per_object.1.iter().any(|(p, m)| *p == emptied_page + && m.map_or(true, |m| m.object_count == 0 && m.allocated_bytes == 0)), + "the chosen page must be emptied, or the reset path went unexercised" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/oldgen.rs b/crates/perry-runtime/src/gc/oldgen.rs index 0c0ef47b5d..0a96aea979 100644 --- a/crates/perry-runtime/src/gc/oldgen.rs +++ b/crates/perry-runtime/src/gc/oldgen.rs @@ -1345,6 +1345,8 @@ impl IncrementalSweepState { struct ArenaSweepObjectsState { cursor: crate::arena::ArenaObjectCursor, + /// Dead old headers awaiting one batched page-index removal (see `sweep_batch`). + pending_old_unregister: sweep_batch::PendingOldUnregister, block_snapshots: Vec, block_has_live: Vec, resettable_general_n: usize, @@ -1404,6 +1406,7 @@ impl ArenaSweepObjectsState { crate::arena::old_pages_reset_sweep_accounting(); Self { cursor: crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::BlockIndex), + pending_old_unregister: Default::default(), block_snapshots, block_has_live: vec![false; n_blocks], resettable_general_n: crate::arena::general_block_count(), @@ -1445,14 +1448,18 @@ impl ArenaSweepObjectsState { fn step(&mut self, budget: usize) -> bool { let mut remaining = budget; + let mut done = false; while remaining > 0 { let Some((header_ptr, block_idx)) = self.cursor.next() else { - return true; + done = true; + break; }; remaining -= 1; self.process_object(header_ptr as *mut GcHeader, block_idx); } - false + // Never leave a dead header in the page index across a step boundary. + self.pending_old_unregister.flush(); + done } fn block_has_live(&self) -> &[bool] { @@ -1624,7 +1631,7 @@ impl ArenaSweepObjectsState { gc_type_clear_dead_payload_side_tables((*header).obj_type, user_ptr as usize); } if self.reclaim_dead_old_blocks && dead_old { - invalidate_dead_old_arena_header(header, total_size); + self.pending_old_unregister.defer(header, total_size); } else { (*header).gc_flags = flags & !(GC_FLAG_FORWARDED | GC_FLAG_MARKED); } @@ -1643,7 +1650,7 @@ impl ArenaSweepObjectsState { } finalize_dead_arena_payload(header, user_ptr, self.overflow_active); if self.reclaim_dead_old_blocks && dead_old { - invalidate_dead_old_arena_header(header, total_size); + self.pending_old_unregister.defer(header, total_size); } } } @@ -1656,6 +1663,7 @@ enum ArenaSweepCleanupSubphase { Done, } +mod sweep_batch; mod sweep_cleanup; use sweep_cleanup::*; diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs new file mode 100644 index 0000000000..db13089b1c --- /dev/null +++ b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs @@ -0,0 +1,46 @@ +//! Batched page-index removal for the dead old objects a sweep step frees. +//! +//! `invalidate_dead_old_arena_header` unregisters each dead old object from the +//! page index on its own, and that removal is quadratic in objects per page (see +//! `arena::unregister_old_objects_batch`). A full collection frees whole pages +//! of small objects, so the sweep queues them here and removes a step's worth in +//! one pass. The header fields are still invalidated immediately, exactly as +//! before, so no walker can read a dead header as a live object in between. + +use super::super::*; + +/// Flush once this many dead headers are queued, so a single unbudgeted sweep +/// of a large heap never stages an unbounded buffer. +const FLUSH_AT: usize = 4096; + +#[derive(Default)] +pub(super) struct PendingOldUnregister { + dead: Vec<(usize, usize)>, + scratch: Vec<(usize, usize, usize)>, +} + +impl PendingOldUnregister { + /// Invalidate a dead old header now and queue its page-index removal. + /// + /// # Safety + /// + /// `header` must be a dead old-gen arena header of `total_size` bytes. + pub(super) unsafe fn defer(&mut self, header: *mut GcHeader, total_size: usize) { + (*header).obj_type = 0; + (*header).gc_flags = 0; + (*header)._reserved = 0; + self.dead.push((header as usize, total_size)); + if self.dead.len() >= FLUSH_AT { + self.flush(); + } + } + + /// Remove every queued header from the page index. + pub(super) fn flush(&mut self) { + if self.dead.is_empty() { + return; + } + crate::arena::unregister_old_objects_batch(&self.dead, &mut self.scratch); + self.dead.clear(); + } +}