perf(gc): batch dead old-object page unregistration per sweep step

crates/perry-runtime/src/arena/mod.rs

@@ -28,6 +28,8 @@ mod walk;
 #[cfg(test)]
 mod tests;
 #[cfg(test)]
+mod tests_batch_unregister;
+#[cfg(test)]
 mod tests_promoted_runs;
 
 // Cross-sibling shared types/thread-locals (used by sibling modules via
@@ -152,8 +154,8 @@ pub(crate) use page_meta::{
     old_page_account_swept_object, old_page_clear_dirty, old_page_mark_dirty,
     old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle,
     old_pages_reset_sweep_accounting, record_arena_object_start, unregister_old_object_pages,
-    HeapGeneration, HeapSpace, OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta,
-    OldPageSummary,
+    unregister_old_objects_batch, HeapGeneration, HeapSpace, OldArenaPageObjectCursor,
+    OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary,
 };
 
 #[cfg(test)]

crates/perry-runtime/src/arena/page_meta/mod.rs

@@ -1285,6 +1285,115 @@ pub(crate) fn unregister_old_object_pages(header_addr: usize, total_size: usize)
     update_old_page_meta_for_object(&removed_pages, false);
 }
 
+/// Batched [`unregister_old_object_pages`] for the dead old objects one sweep
+/// step found.
+///
+/// The per-object remover allocates two `Vec`s, flushes the deferral buffer,
+/// borrows both tables, and then finds the header in its page's object list
+/// with a linear `position` before `swap_remove`. Freeing every object on a
+/// page that way is quadratic in objects per page, and a full collection frees
+/// whole pages of small objects: on records_array_8m:scan (~720k dead records
+/// per full) it was 15.8% of ALL samples, the single largest cost of a full
+/// collection. The registration side already batches for the same reason
+/// (`flush_deferred_old_page_registrations_batch`); this is its mirror.
+///
+/// One flush, one run materialization per touched page, then one `retain` per
+/// page against that page's dead headers (sorted, so membership is a binary
+/// search). Page-meta decrements for a page are applied together and its
+/// reset/refresh run once: allocated bytes and object counts only fall here,
+/// so "both reached zero at some point" and "both are zero at the end" are the
+/// same event, and `reset_cycle_sweep_accounting` / `refresh_policy_bits` are
+/// pure recomputes of the page's own fields.
+///
+/// `scratch` is caller-owned and reused across steps, so a flush allocates
+/// nothing once warm (the #7624 lesson: re-growing staging buffers per batch
+/// cost +31 MB peak RSS on json_pipeline).
+pub(crate) fn unregister_old_objects_batch(
+    dead: &[(usize, usize)],
+    scratch: &mut Vec<(usize, usize, usize)>,
+) {
+    if dead.is_empty() {
+        return;
+    }
+    flush_deferred_old_page_registrations();
+    scratch.clear();
+    for &(header_addr, total_size) in dead {
+        if header_addr == 0 || total_size == 0 {
+            continue;
+        }
+        let object_end = header_addr + total_size;
+        let first_page = generation_page_for_addr(header_addr);
+        let last_page = generation_page_for_addr(object_end - 1);
+        for page in first_page..=last_page {
+            let page_base = generation_page_base(page);
+            let overlap_start = header_addr.max(page_base);
+            let overlap_end = object_end.min(page_base + GENERATION_PAGE_SIZE);
+            if overlap_start < overlap_end {
+                scratch.push((page, header_addr, overlap_end - overlap_start));
+            }
+        }
+    }
+    // RUN REMOVER, as the per-object path: expand before touching membership.
+    // Done before borrowing the index, because expansion writes it.
+    if OLD_GEN_PAGE_PROMOTED_RUNS_NONEMPTY.with(Cell::get) {
+        let mut last = usize::MAX;
+        for &(page, _, _) in scratch.iter() {
+            if page != last {
+                materialize_promoted_page_runs(core::iter::once(page));
+                last = page;
+            }
+        }
+    }
+    scratch.sort_unstable_by_key(|&(page, header, _)| (page, header));
+    OLD_GEN_PAGE_OBJECTS.with(|index| {
+        OLD_GEN_PAGE_META.with(|meta| {
+            let mut index = index.borrow_mut();
+            let mut meta = meta.borrow_mut();
+            let mut start = 0;
+            while start < scratch.len() {
+                let page = scratch[start].0;
+                let mut end = start;
+                while end < scratch.len() && scratch[end].0 == page {
+                    end += 1;
+                }
+                let group = &scratch[start..end];
+                let mut removed_bytes = 0usize;
+                let mut removed_objects = 0usize;
+                let mut remove_page = false;
+                if let Some(headers) = index.get_mut(&page) {
+                    headers.retain(|&addr| {
+                        match group.binary_search_by_key(&addr, |&(_, header, _)| header) {
+                            Ok(i) => {
+                                removed_bytes = removed_bytes.saturating_add(group[i].2);
+                                removed_objects += 1;
+                                false
+                            }
+                            Err(_) => true,
+                        }
+                    });
+                    remove_page = headers.is_empty();
+                }
+                if remove_page {
+                    index.remove(&page);
+                }
+                if removed_objects != 0 {
+                    let page_meta = meta
+                        .entry(page)
+                        .or_insert_with(|| OldPageMeta::zero_for_page(page));
+                    page_meta.allocated_bytes =
+                        page_meta.allocated_bytes.saturating_sub(removed_bytes);
+                    page_meta.object_count = page_meta.object_count.saturating_sub(removed_objects);
+                    if page_meta.allocated_bytes == 0 && page_meta.object_count == 0 {
+                        page_meta.reset_cycle_sweep_accounting();
+                    }
+                    page_meta.refresh_policy_bits();
+                }
+                start = end;
+            }
+        });
+    });
+}
+
 pub(crate) fn old_pages_begin_gc_cycle() {
     // #7624 CYCLE START: all three cycle constructors route through here
     // (`gc/mod.rs`'s minor, `gc/cycle.rs`'s `new_full`, `gc/policy.rs`'s

crates/perry-runtime/src/arena/tests.rs

@@ -211,7 +211,7 @@ fn old_page_meta(page: usize) -> OldPageMeta {
     old_page_meta_for_tests(page).expect("old page metadata should be registered")
 }
 
-fn old_header_and_size(user_ptr: usize) -> (usize, usize) {
+pub(super) fn old_header_and_size(user_ptr: usize) -> (usize, usize) {
     let header_addr = user_ptr - GC_HEADER_SIZE;
     let total_size = unsafe { (*(header_addr as *const GcHeader)).size as usize };
     (header_addr, total_size)

crates/perry-runtime/src/arena/tests_batch_unregister.rs

@@ -0,0 +1,123 @@
+//! The batched old-object page unregister must be indistinguishable from the
+//! per-object remover it replaces on the full sweep's hot path.
+//!
+//! Both are driven over the SAME population in one fresh arena: remove a subset
+//! one object at a time and snapshot, restore the subset, remove it again in
+//! one batch and snapshot, then require the two snapshots to be identical --
+//! page membership as walked, and every touched page's metadata. The subset
+//! mixes page-spanning objects, fully emptied pages and partial pages, because
+//! those are the three shapes the grouping and the per-page reset/refresh
+//! have to get right.
+
+use super::page_meta::{
+    old_arena_walk_objects_on_pages, old_object_page_overlaps, old_page_meta_for_tests,
+    register_old_object_pages, unregister_old_object_pages, unregister_old_objects_batch,
+    OldPageMeta, GENERATION_PAGE_SIZE,
+};
+use super::tests::old_header_and_size as header_and_size;
+use super::*;
+use crate::gc::GC_TYPE_STRING;
+
+type Snapshot = (Vec<usize>, Vec<(usize, Option<OldPageMeta>)>);
+
+fn snapshot(pages: &[usize]) -> Snapshot {
+    let mut set = crate::fast_hash::new_ptr_hash_set();
+    for &page in pages {
+        set.insert(page);
+    }
+    let mut members = Vec::new();
+    old_arena_walk_objects_on_pages(&set, |header| members.push(header as usize));
+    members.sort_unstable();
+    let metas = pages
+        .iter()
+        .map(|&page| (page, old_page_meta_for_tests(page)))
+        .collect();
+    (members, metas)
+}
+
+#[test]
+fn batched_unregister_leaves_the_same_index_and_metadata_as_per_object_removal() {
+    super::tests::run_with_fresh_arenas(|| {
+        // Many small objects per page, plus objects wider than a page.
+        let mut objects = Vec::new();
+        for i in 0..600 {
+            let size = if i % 97 == 0 {
+                GENERATION_PAGE_SIZE + 512
+            } else {
+                72
+            };
+            let ptr = arena_alloc_gc_old(size, 8, GC_TYPE_STRING) as usize;
+            objects.push(header_and_size(ptr));
+        }
+        let mut pages: Vec<usize> = objects
+            .iter()
+            .flat_map(|&(h, s)| old_object_page_overlaps(h, s).into_iter().map(|(p, _)| p))
+            .collect();
+        pages.sort_unstable();
+        pages.dedup();
+
+        // Every third object, every page-spanning object, and EVERY object on
+        // one chosen page, so at least one page empties completely.
+        let emptied_page = pages[pages.len() / 2];
+        let subset: Vec<(usize, usize)> = objects
+            .iter()
+            .enumerate()
+            .filter(|&(i, &(h, s))| {
+                i % 3 == 0
+                    || s > GENERATION_PAGE_SIZE
+                    || old_object_page_overlaps(h, s)
+                        .iter()
+                        .any(|&(p, _)| p == emptied_page)
+            })
+            .map(|(_, &o)| o)
+            .collect();
+        assert!(subset.len() > 200 && subset.len() < objects.len());
+
+        let before = snapshot(&pages);
+
+        for &(h, s) in &subset {
+            unregister_old_object_pages(h, s);
+        }
+        let per_object = snapshot(&pages);
+        assert_ne!(per_object, before, "the subset must actually be removed");
+
+        for &(h, s) in &subset {
+            register_old_object_pages(h, s);
+        }
+        assert_eq!(
+            snapshot(&pages).0,
+            before.0,
+            "restoring the subset must restore page membership"
+        );
+
+        let mut scratch = Vec::new();
+        unregister_old_objects_batch(&subset, &mut scratch);
+        let batched = snapshot(&pages);
+
+        assert_eq!(
+            batched.0, per_object.0,
+            "batched removal must leave exactly the headers per-object removal leaves"
+        );
+        for ((page, a), (_, b)) in per_object.1.iter().zip(batched.1.iter()) {
+            let fields = |m: &Option<OldPageMeta>| {
+                m.map(|m| {
+                    (
+                        m.allocated_bytes,
+                        m.object_count,
+                        m.live_bytes,
+                        m.dead_bytes,
+                        m.live_object_count,
+                        m.dead_object_count,
+                        m.evacuation_eligible,
+                    )
+                })
+            };
+            assert_eq!(fields(a), fields(b), "page {page:#x} metadata diverged");
+        }
+        assert!(
+            per_object.1.iter().any(|(p, m)| *p == emptied_page
+                && m.map_or(true, |m| m.object_count == 0 && m.allocated_bytes == 0)),
+            "the chosen page must be emptied, or the reset path went unexercised"
+        );
+    });
+}

crates/perry-runtime/src/gc/oldgen.rs

@@ -1345,6 +1345,8 @@ impl IncrementalSweepState {
 
 struct ArenaSweepObjectsState {
     cursor: crate::arena::ArenaObjectCursor,
+    /// Dead old headers awaiting one batched page-index removal (see `sweep_batch`).
+    pending_old_unregister: sweep_batch::PendingOldUnregister,
     block_snapshots: Vec<crate::arena::ArenaBlockSnapshot>,
     block_has_live: Vec<bool>,
     resettable_general_n: usize,
@@ -1404,6 +1406,7 @@ impl ArenaSweepObjectsState {
         crate::arena::old_pages_reset_sweep_accounting();
         Self {
             cursor: crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::BlockIndex),
+            pending_old_unregister: Default::default(),
             block_snapshots,
             block_has_live: vec![false; n_blocks],
             resettable_general_n: crate::arena::general_block_count(),
@@ -1445,14 +1448,18 @@ impl ArenaSweepObjectsState {
 
     fn step(&mut self, budget: usize) -> bool {
         let mut remaining = budget;
+        let mut done = false;
         while remaining > 0 {
             let Some((header_ptr, block_idx)) = self.cursor.next() else {
-                return true;
+                done = true;
+                break;
             };
             remaining -= 1;
             self.process_object(header_ptr as *mut GcHeader, block_idx);
         }
-        false
+        // Never leave a dead header in the page index across a step boundary.
+        self.pending_old_unregister.flush();
+        done
     }
 
     fn block_has_live(&self) -> &[bool] {
@@ -1624,7 +1631,7 @@ impl ArenaSweepObjectsState {
             gc_type_clear_dead_payload_side_tables((*header).obj_type, user_ptr as usize);
         }
         if self.reclaim_dead_old_blocks && dead_old {
-            invalidate_dead_old_arena_header(header, total_size);
+            self.pending_old_unregister.defer(header, total_size);
         } else {
             (*header).gc_flags = flags & !(GC_FLAG_FORWARDED | GC_FLAG_MARKED);
         }
@@ -1643,7 +1650,7 @@ impl ArenaSweepObjectsState {
         }
         finalize_dead_arena_payload(header, user_ptr, self.overflow_active);
         if self.reclaim_dead_old_blocks && dead_old {
-            invalidate_dead_old_arena_header(header, total_size);
+            self.pending_old_unregister.defer(header, total_size);
         }
     }
 }
@@ -1656,6 +1663,7 @@ enum ArenaSweepCleanupSubphase {
     Done,
 }
 
+mod sweep_batch;
 mod sweep_cleanup;
 use sweep_cleanup::*;
 

crates/perry-runtime/src/gc/oldgen/sweep_batch.rs

@@ -0,0 +1,46 @@
+//! Batched page-index removal for the dead old objects a sweep step frees.
+//!
+//! `invalidate_dead_old_arena_header` unregisters each dead old object from the
+//! page index on its own, and that removal is quadratic in objects per page (see
+//! `arena::unregister_old_objects_batch`). A full collection frees whole pages
+//! of small objects, so the sweep queues them here and removes a step's worth in
+//! one pass. The header fields are still invalidated immediately, exactly as
+//! before, so no walker can read a dead header as a live object in between.
+
+use super::super::*;
+
+/// Flush once this many dead headers are queued, so a single unbudgeted sweep
+/// of a large heap never stages an unbounded buffer.
+const FLUSH_AT: usize = 4096;
+
+#[derive(Default)]
+pub(super) struct PendingOldUnregister {
+    dead: Vec<(usize, usize)>,
+    scratch: Vec<(usize, usize, usize)>,
+}
+
+impl PendingOldUnregister {
+    /// Invalidate a dead old header now and queue its page-index removal.
+    ///
+    /// # Safety
+    ///
+    /// `header` must be a dead old-gen arena header of `total_size` bytes.
+    pub(super) unsafe fn defer(&mut self, header: *mut GcHeader, total_size: usize) {
+        (*header).obj_type = 0;
+        (*header).gc_flags = 0;
+        (*header)._reserved = 0;
+        self.dead.push((header as usize, total_size));
+        if self.dead.len() >= FLUSH_AT {
+            self.flush();
+        }
+    }
+
+    /// Remove every queued header from the page index.
+    pub(super) fn flush(&mut self) {
+        if self.dead.is_empty() {
+            return;
+        }
+        crate::arena::unregister_old_objects_batch(&self.dead, &mut self.scratch);
+        self.dead.clear();
+    }
+}