gc: make a synchronous full cheaper per live object (#10182) [pacing retry does not meet acceptance]

changelog.d/10220-gc-full-throughput.md

@@ -0,0 +1,36 @@
+A synchronous full mark-sweep costs less per live object, in three exact
+changes:
+
+- **Census membership from per-block start bitmaps.** `ValidPointerSet` used
+  to answer "is this an arena object start?" with two binary searches (over
+  every 1024-start run's first key, then inside the run), about 46 % of a full
+  mark on a live JSON tree. The census now records one object-start bitmap per
+  block it walks (one bit per 8-byte alignment unit, allocated in 8 KiB chunks;
+  oversized blocks keep a sorted start list), so a traced pointer field costs a
+  search over the block fences and one bit test. Interior-pointer lookups scan
+  the bitmap backwards. The bitmaps are 1/64 of the walked bytes instead of
+  8 bytes per censused object. A single contiguous bitmap vector grew into the
+  allocator's large-page class and cost `records_array_1m:sparse` +4.8 MiB
+  peak RSS for a ~100 KB index; the chunked storage reads 68 MiB, as before.
+- **No remembered-set rebuild when nothing young is live.** When the census
+  shows no marked or pinned young object after the mark and the malloc
+  registry is empty, the old-to-young rebuild can only produce an empty set,
+  so the full installs one without walking the old generation.
+  `PERRY_GC_DIAG=1` prints `[gc-remembered-rebuild] full
+  skipped=young_generation_unmarked`.
+- **Sweep page accounting once per page.** Consecutive single-page old objects
+  are summed and applied to their page metadata once, always before a
+  page-index flush that could zero that page's accounting, instead of one
+  overlap vector allocation and one hash lookup per object.
+
+On a loop that keeps one parsed `records_array_20m.json` tree in the old
+generation and calls `gc()`, a full drops from 82–83 ms to 37–39 ms: mark
+43 → 25 ms, rebuild 21 → 0 ms, sweep 13 → 7 ms, census 5 ms unchanged; peak
+footprint 129 → 119 MiB. The 22-row JSON matrix stays within ±2 % CPU with no
+row worse on RSS, and the gc-ratchet gated counters are identical.
+
+Retrying the promoted-cohort pacing bound with these costs still does not
+meet the #10182 bar: at `k=1, floor 16 MB` the 20 MB parse/scan/sparse rows
+reach 179 MiB (below Node's 220) but cost 266–275 ms of CPU against a 208 ms
+best, and `records_array_8m:scan` has 13.7 ms of CPU headroom for 8
+iterations, less than one ~30 ms full over its tree. No pacing change is included.

crates/perry-runtime/src/arena/mod.rs

@@ -102,8 +102,8 @@ pub use walk::{
 pub(crate) use walk::{
     arena_block_snapshots, arena_telemetry_snapshot, general_block_in_recent_window,
     general_block_sizes, old_arena_walk_all_headers_filtered, young_allocation_census,
-    ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder, ArenaTelemetrySnapshot,
-    ArenaWalkOrder,
+    young_block_count, ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder,
+    ArenaTelemetrySnapshot, ArenaWalkOrder,
 };
 
 // reset.rs
@@ -150,12 +150,13 @@ pub(crate) use page_meta::{
     classify_heap_space_in_range, generation_page_for_addr, materialize_all_promoted_page_runs,
     old_arena_block_range_index, old_arena_block_ranges, old_arena_page_index_remove_object,
     old_arena_source_blocks_for_pages, old_arena_walk_objects_on_pages, old_object_page_overlaps,
-    old_page_account_dirty_slot, old_page_account_dirty_slots, old_page_account_promoted_object,
-    old_page_account_swept_object, old_page_clear_dirty, old_page_mark_dirty,
-    old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle,
-    old_pages_reset_sweep_accounting, record_arena_object_start, unregister_old_object_pages,
-    unregister_old_objects_batch, HeapGeneration, HeapSpace, OldArenaPageObjectCursor,
-    OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary,
+    old_object_single_page, old_page_account_dirty_slot, old_page_account_dirty_slots,
+    old_page_account_promoted_object, old_page_account_swept_object, old_page_account_swept_tally,
+    old_page_clear_dirty, old_page_mark_dirty, old_page_meta_snapshot, old_page_summary,
+    old_pages_begin_gc_cycle, old_pages_reset_sweep_accounting, record_arena_object_start,
+    unregister_old_object_pages, unregister_old_objects_batch, HeapGeneration, HeapSpace,
+    OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary,
+    OldPageSweepTally,
 };
 
 #[cfg(test)]

crates/perry-runtime/src/arena/page_meta/mod.rs

@@ -139,7 +139,11 @@ impl PageGenerationCache {
 const PAGE_GENERATION_CACHE_WAYS: usize = 4;
 
 mod page_class;
+mod sweep_tally;
 pub(crate) use page_class::*;
+pub(crate) use sweep_tally::{
+    old_object_single_page, old_page_account_swept_tally, OldPageSweepTally,
+};
 
 #[cfg(test)]
 mod tests;

crates/perry-runtime/src/arena/page_meta/sweep_tally.rs

@@ -0,0 +1,87 @@
+//! Per-page batching of a sweep's old-generation page accounting (#10182).
+//!
+//! `old_page_account_swept_object` costs one heap-allocated overlap vector and
+//! one page-meta hash lookup per swept old object. A full sweep over a live
+//! 20 MB JSON tree calls it ~585k times, and it was about three quarters of the
+//! sweep. The sweep walks each block in address order, so consecutive objects
+//! share a page: summing their deltas and applying the sum once per page gives
+//! the same page meta, because every field involved is a plain sum and
+//! `refresh_policy_bits` is a pure recompute of the page's own fields.
+//!
+//! The one interleaving that could observe the difference is the batched
+//! page-index removal (`unregister_old_objects_batch`), which zeroes a page's
+//! sweep accounting when its last object leaves. The sweep therefore applies
+//! its tally before every such flush, which keeps the order of resets and sums
+//! on every page exactly as the one-by-one calls had it.
+
+use super::*;
+
+/// Sweep accounting deltas of consecutive old objects that each lie wholly on
+/// one page.
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub(crate) struct OldPageSweepTally {
+    pub(crate) live_bytes: usize,
+    pub(crate) live_objects: usize,
+    pub(crate) pinned_bytes: usize,
+    pub(crate) pinned_objects: usize,
+    pub(crate) dead_bytes: usize,
+    pub(crate) dead_objects: usize,
+}
+
+impl OldPageSweepTally {
+    #[inline(always)]
+    pub(crate) fn add(&mut self, total_size: usize, live: bool, pinned: bool) {
+        if live {
+            self.live_bytes += total_size;
+            self.live_objects += 1;
+            if pinned {
+                self.pinned_bytes += total_size;
+                self.pinned_objects += 1;
+            }
+        } else {
+            self.dead_bytes += total_size;
+            self.dead_objects += 1;
+        }
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.live_objects == 0 && self.dead_objects == 0
+    }
+}
+
+/// The page an old object lies on when it lies wholly on one page.
+#[inline(always)]
+pub(crate) fn old_object_single_page(header_addr: usize, total_size: usize) -> Option<usize> {
+    if header_addr == 0 || total_size == 0 {
+        return None;
+    }
+    let first_page = generation_page_for_addr(header_addr);
+    (first_page == generation_page_for_addr(header_addr + total_size - 1)).then_some(first_page)
+}
+
+/// Apply `tally` to `page` exactly as the same objects' one-by-one
+/// `old_page_account_swept_object` calls would have.
+pub(crate) fn old_page_account_swept_tally(page: usize, tally: &OldPageSweepTally) {
+    if tally.is_empty() {
+        return;
+    }
+    OLD_GEN_PAGE_META.with(|meta| {
+        let mut meta = meta.borrow_mut();
+        let page_meta = meta
+            .entry(page)
+            .or_insert_with(|| OldPageMeta::zero_for_page(page));
+        page_meta.live_bytes = page_meta.live_bytes.saturating_add(tally.live_bytes);
+        page_meta.live_object_count = page_meta
+            .live_object_count
+            .saturating_add(tally.live_objects);
+        page_meta.pinned_bytes = page_meta.pinned_bytes.saturating_add(tally.pinned_bytes);
+        page_meta.pinned_object_count = page_meta
+            .pinned_object_count
+            .saturating_add(tally.pinned_objects);
+        page_meta.dead_bytes = page_meta.dead_bytes.saturating_add(tally.dead_bytes);
+        page_meta.dead_object_count = page_meta
+            .dead_object_count
+            .saturating_add(tally.dead_objects);
+        page_meta.refresh_policy_bits();
+    });
+}

crates/perry-runtime/src/arena/walk.rs

@@ -905,6 +905,16 @@ pub fn general_block_count() -> usize {
     ARENA.with(|arena| unsafe { (*arena.get()).blocks.len() })
 }
 
+/// Global block indices `0..young_block_count()` are the young generation:
+/// the general (Eden) arena's blocks, then both survivor semispaces' — the
+/// order `arena_block_snapshots` and the object cursors use.
+pub(crate) fn young_block_count() -> usize {
+    let g = ARENA.with(|arena| unsafe { (*arena.get()).blocks.len() });
+    let s0 = SURVIVOR_ARENA_0.with(|arena| unsafe { (*arena.get()).blocks.len() });
+    let s1 = SURVIVOR_ARENA_1.with(|arena| unsafe { (*arena.get()).blocks.len() });
+    g + s0 + s1
+}
+
 /// Per-block `size` for the general (nursery-Eden) arena, indexed by
 /// general block index (`0..general_block_count()`, tombstones report
 /// size 0). Used by the evacuation policy to translate "this block's

crates/perry-runtime/src/gc/cycle.rs

@@ -1294,6 +1294,15 @@ impl GcCycleState {
                         .as_mut()
                         .expect("atomic finalize state exists");
                     let rebuild = state.remembered_rebuild.get_or_insert_with(|| {
+                        // #10182: nothing young is marked and no malloc object
+                        // exists, so the walk could only insert nothing.
+                        if !budgeted
+                            && valid_ptrs.is_some_and(|ptrs| {
+                                full_remembered_rebuild_provably_empty(&ptrs.block_census)
+                            })
+                        {
+                            return OldToYoungRememberedRebuildState::provably_empty();
+                        }
                         let skip = if budgeted {
                             None
                         } else {

crates/perry-runtime/src/gc/oldgen/sweep_batch.rs

@@ -35,6 +35,13 @@ impl PendingOldUnregister {
         }
     }
 
+    /// Will the next `defer` flush the queue (and so zero the sweep accounting
+    /// of every page whose last object it removes)?
+    #[inline]
+    pub(super) fn flushes_on_next_defer(&self) -> bool {
+        self.dead.len() + 1 >= FLUSH_AT
+    }
+
     /// Remove every queued header from the page index.
     pub(super) fn flush(&mut self) {
         if self.dead.is_empty() {

crates/perry-runtime/src/gc/oldgen/sweep_objects.rs

@@ -7,6 +7,10 @@ pub(super) struct ArenaSweepObjectsState {
     cursor: crate::arena::ArenaObjectCursor,
     /// Dead old headers awaiting one batched page-index removal (see `sweep_batch`).
     pending_old_unregister: super::sweep_batch::PendingOldUnregister,
+    /// Page accounting of consecutive single-page old objects, applied once per
+    /// page (see `arena::page_meta::sweep_tally`). `usize::MAX` when empty.
+    old_page_tally_page: usize,
+    old_page_tally: crate::arena::OldPageSweepTally,
     block_snapshots: Vec<crate::arena::ArenaBlockSnapshot>,
     block_has_live: Vec<bool>,
     resettable_general_n: usize,
@@ -71,6 +75,8 @@ impl ArenaSweepObjectsState {
         Self {
             cursor: crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::BlockIndex),
             pending_old_unregister: Default::default(),
+            old_page_tally_page: usize::MAX,
+            old_page_tally: Default::default(),
             block_snapshots,
             block_has_live: vec![false; n_blocks],
             resettable_general_n: crate::arena::general_block_count(),
@@ -218,11 +224,79 @@ impl ArenaSweepObjectsState {
             remaining -= 1;
             self.process_object(header_ptr as *mut GcHeader, block_idx);
         }
-        // Never leave a dead header in the page index across a step boundary.
-        self.pending_old_unregister.flush();
+        // Never leave a dead header in the page index, or a page's accounting
+        // unapplied, across a step boundary. The tally goes first: the
+        // unregister flush zeroes the accounting of pages it empties.
+        if self.page_tally_order_kept() {
+            self.apply_old_page_tally();
+            self.pending_old_unregister.flush();
+        } else {
+            self.pending_old_unregister.flush();
+            self.apply_old_page_tally();
+        }
         done
     }
 
+    /// Account one swept old object on its page(s), batching single-page
+    /// objects per page.
+    #[inline]
+    fn account_old_object(
+        &mut self,
+        header: *mut GcHeader,
+        total_size: usize,
+        live: bool,
+        pinned: bool,
+    ) {
+        match crate::arena::old_object_single_page(header as usize, total_size) {
+            Some(page) => {
+                if page != self.old_page_tally_page {
+                    self.apply_old_page_tally();
+                    self.old_page_tally_page = page;
+                }
+                self.old_page_tally.add(total_size, live, pinned);
+            }
+            None => crate::arena::old_page_account_swept_object(
+                header as usize,
+                total_size,
+                live,
+                pinned,
+            ),
+        }
+    }
+
+    fn apply_old_page_tally(&mut self) {
+        if self.old_page_tally_page != usize::MAX {
+            crate::arena::old_page_account_swept_tally(
+                self.old_page_tally_page,
+                &self.old_page_tally,
+            );
+        }
+        self.old_page_tally_page = usize::MAX;
+        self.old_page_tally = Default::default();
+    }
+
+    /// The tally is applied before every page-index flush (always, outside the
+    /// sabotaged test).
+    #[inline(always)]
+    fn page_tally_order_kept(&self) -> bool {
+        #[cfg(test)]
+        {
+            use super::super::trace::block_skip::sabotage;
+            sabotage::get() & sabotage::FORGET_PAGE_TALLY_ORDER == 0
+        }
+        #[cfg(not(test))]
+        true
+    }
+
+    /// Queue a dead old header's page-index removal, applying the page tally
+    /// first when the queue is about to flush.
+    unsafe fn defer_old_unregister(&mut self, header: *mut GcHeader, total_size: usize) {
+        if self.pending_old_unregister.flushes_on_next_defer() && self.page_tally_order_kept() {
+            self.apply_old_page_tally();
+        }
+        self.pending_old_unregister.defer(header, total_size);
+    }
+
     pub(super) fn block_has_live(&self) -> &[bool] {
         &self.block_has_live
     }
@@ -315,12 +389,7 @@ impl ArenaSweepObjectsState {
         count_in_live_census: bool,
     ) {
         if block_idx >= self.old_block_start {
-            crate::arena::old_page_account_swept_object(
-                header as usize,
-                (*header).size as usize,
-                true,
-                pinned,
-            );
+            self.account_old_object(header, (*header).size as usize, true, pinned);
         }
         if block_idx < self.block_has_live.len() {
             self.block_has_live[block_idx] = true;
@@ -386,7 +455,7 @@ impl ArenaSweepObjectsState {
         let total_size = (*header).size as usize;
         let dead_old = block_idx >= self.old_block_start;
         if dead_old {
-            crate::arena::old_page_account_swept_object(header as usize, total_size, false, false);
+            self.account_old_object(header, total_size, false, false);
         }
         let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE);
         self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64);
@@ -395,7 +464,7 @@ impl ArenaSweepObjectsState {
             gc_type_clear_dead_payload_side_tables((*header).obj_type, user_ptr as usize);
         }
         if self.reclaim_dead_old_blocks && dead_old {
-            self.pending_old_unregister.defer(header, total_size);
+            self.defer_old_unregister(header, total_size);
         } else {
             (*header).gc_flags = flags & !(GC_FLAG_FORWARDED | GC_FLAG_MARKED);
         }
@@ -405,7 +474,7 @@ impl ArenaSweepObjectsState {
         let total_size = (*header).size as usize;
         let dead_old = block_idx >= self.old_block_start;
         if dead_old {
-            crate::arena::old_page_account_swept_object(header as usize, total_size, false, false);
+            self.account_old_object(header, total_size, false, false);
         }
         let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE);
         self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64);
@@ -414,7 +483,7 @@ impl ArenaSweepObjectsState {
         }
         finalize_dead_arena_payload(header, user_ptr, self.overflow_active);
         if self.reclaim_dead_old_blocks && dead_old {
-            self.pending_old_unregister.defer(header, total_size);
+            self.defer_old_unregister(header, total_size);
         }
     }
 }