From 0a26e35e34c008d895086eb3eaeabd9e0f219d0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:15:54 +0200 Subject: [PATCH 01/11] gc: answer census arena membership from per-block object-start bitmaps (#10182) --- .../perry-runtime/src/gc/tests/cycle_state.rs | 51 ++- crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/start_bitmap.rs | 214 ++++++++++ crates/perry-runtime/src/gc/trace.rs | 398 ++++++++++++------ 4 files changed, 501 insertions(+), 163 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/start_bitmap.rs diff --git a/crates/perry-runtime/src/gc/tests/cycle_state.rs b/crates/perry-runtime/src/gc/tests/cycle_state.rs index 2b5d531fc6..4fa2daa7cf 100644 --- a/crates/perry-runtime/src/gc/tests/cycle_state.rs +++ b/crates/perry-runtime/src/gc/tests/cycle_state.rs @@ -271,13 +271,13 @@ fn build_valid_pointer_set_sliced_build_preserves_contains_and_enclosing_object( } } -/// #7646: arena membership now answers from the address-ordered census runs -/// rather than a shadow `BTreeSet`, which makes RUN BOUNDARIES load-bearing. -/// Runs seal every `VALID_POINTER_ARENA_RUN_CAPACITY` (1024) starts, so the -/// final run is partial and is only sealed by `finalize()`. The sliced-build -/// test above allocates 1100 strings — enough to cross the boundary — but -/// checks only the first 16, which all live in the FIRST run: it passes -/// unchanged if every later run is lost. +/// #7646: arena membership answers from the census's own address-ordered +/// structure rather than a shadow `BTreeSet`, which makes the structure's +/// boundaries load-bearing. It was a list of 1024-start runs whose last, partial +/// run only `finalize()` sealed; since #10182 it is one start bitmap per censused +/// block, where a lost block entry or a stale fence would drop every start of +/// that block. The sliced-build test above checks only the first 16 starts, so +/// it passes unchanged if every later start is lost. /// /// This checks every start, both directions. #[test] @@ -285,8 +285,6 @@ fn valid_pointer_membership_spans_every_census_run_including_the_partial_one_764 let _guard = CopyingNurseryTestGuard::new(0); let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); - // > 2 full runs, so the last one is partial and cannot be sealed by the - // capacity check alone. let arena_strings = (0..2600).map(|_| young_leaf()).collect::>(); let (arena_object, fields) = unsafe { alloc_nursery_test_object(4) }; let arena_object = arena_object as usize; @@ -295,30 +293,29 @@ fn valid_pointer_membership_spans_every_census_run_including_the_partial_one_764 let valid_ptrs = ValidPointerSetBuilder::new().finish(); assert!( - valid_ptrs.arena_runs.len() >= 3, - "premise: the census must span several runs, got {}", - valid_ptrs.arena_runs.len() - ); - assert!( - valid_ptrs.current_arena_run.is_empty(), - "finalize() must seal the open run before the set escapes the builder; \ - {} starts would otherwise be invisible to membership", - valid_ptrs.current_arena_run.len() + !valid_ptrs.arena_blocks.is_empty(), + "premise: the census must have opened at least one block" ); assert_eq!( - valid_ptrs.arena_run_firsts.len(), - valid_ptrs.arena_runs.len(), - "the fence mirror must stay index-aligned with the runs" + valid_ptrs.arena_block_bases.len(), + valid_ptrs.arena_blocks.len(), + "the fence mirror must stay index-aligned with the census blocks" ); - for (index, run) in valid_ptrs.arena_runs.iter().enumerate() { + for (index, block) in valid_ptrs.arena_blocks.iter().enumerate() { assert_eq!( - valid_ptrs.arena_run_firsts[index], run[0], - "fence {index} must equal its run's first key" + valid_ptrs.arena_block_bases[index], block.base, + "fence {index} must equal its block's base" ); + if index > 0 { + let previous = valid_ptrs.arena_blocks[index - 1]; + assert!( + previous.base + previous.extent <= block.base, + "census blocks must be ascending and disjoint" + ); + } } - // Positive: EVERY censused start, not a prefix — a start in the last, - // partial run is the one a lost `finalize()` drops. + // Positive: EVERY censused start, not a prefix. for (index, &ptr) in arena_strings.iter().enumerate() { assert!( valid_ptrs.contains(&ptr), @@ -372,7 +369,7 @@ fn build_valid_pointer_set_finalize_is_separate_bounded_phase() { } let before_finalize = builder.snapshot_for_tests(); assert_eq!(before_finalize.phase, ValidPointerSetBuildPhase::Finalize); - assert!(before_finalize.current_arena_run_len > 0 || before_finalize.arena_run_count > 0); + assert!(before_finalize.arena_block_count > 0); assert!(!builder.step(0)); assert_eq!( diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index ec960abbbc..f09c1ec962 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -64,6 +64,7 @@ mod shadow_stack_ops; mod shape_descriptor_authority; mod shape_keys_descriptor_edge; mod smoke; +mod start_bitmap; mod step_bounds; pub(super) mod support; mod survival_diag; diff --git a/crates/perry-runtime/src/gc/tests/start_bitmap.rs b/crates/perry-runtime/src/gc/tests/start_bitmap.rs new file mode 100644 index 0000000000..05c5647a41 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/start_bitmap.rs @@ -0,0 +1,214 @@ +//! #10182: the census answers arena membership from one object-start bitmap +//! per censused block (sorted start lists for oversized blocks). +//! +//! Every case plants a population on a fresh thread (so the arenas start +//! empty), builds the production census, and compares `contains` and +//! `enclosing_object` against an oracle derived from an independent arena walk +//! for every address the census covers. The sabotaged twin shifts every bitmap +//! probe by one alignment unit and shows the comparison notices. + +use super::super::*; +use super::support::*; +use crate::gc::trace::start_bitmap_sabotage; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("start-bitmap test thread must not panic"); +} + +/// Plant a population that exercises the bitmap's edges: runs of minimal +/// objects (consecutive bits across word boundaries), objects of assorted and +/// odd sizes, a 600 KB object inside a 1 MB block (an interior pointer deep in +/// it floors across many zero words), an oversized block (sorted start list), +/// nursery leaves and nursery objects. +unsafe fn plant_population() -> Vec { + let mut planted = Vec::new(); + for i in 0..9000usize { + let payload = match i % 7 { + 0 | 1 | 2 => 0, + 3 => 8, + 4 => 13, + 5 => 40, + _ => 200 + (i % 5) * 64, + }; + planted.push(crate::arena::arena_alloc_gc_old(payload, 8, GC_TYPE_STRING) as usize); + } + planted.push(crate::arena::arena_alloc_gc_old(600 * 1024, 8, GC_TYPE_STRING) as usize); + for _ in 0..200 { + planted.push(crate::arena::arena_alloc_gc_old(24, 8, GC_TYPE_STRING) as usize); + } + planted.push(crate::arena::arena_alloc_gc_old( + 2 * crate::arena::BLOCK_SIZE + 4096, + 8, + GC_TYPE_STRING, + ) as usize); + for _ in 0..3000 { + planted.push(young_leaf()); + planted.push(alloc_nursery_test_object(3).0 as usize); + } + planted +} + +/// `(user pointer, total size)` of every walkable arena object, ascending. +fn oracle_objects() -> Vec<(usize, usize)> { + let mut cursor = crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::Address); + let mut objects = Vec::new(); + while let Some((header, _)) = cursor.next() { + let size = unsafe { (*(header as *const GcHeader)).size as usize }; + objects.push((header as usize + GC_HEADER_SIZE, size)); + } + objects +} + +#[derive(Default, Debug)] +struct Comparison { + queries: usize, + contains_mismatches: usize, + enclosing_mismatches: usize, + enclosing_hits: usize, + /// An enclosing hit whose start lies at least one bitmap word (64 units) + /// below the query: the floor search had to cross zero words. + enclosing_hits_across_words: usize, + sorted_blocks: usize, + bitmap_blocks: usize, +} + +/// Query every address (step 4, so both aligned and unaligned ones) from each +/// census block's base to 64 bytes past its walked extent. +fn compare_with_oracle(valid: &ValidPointerSet, objects: &[(usize, usize)]) -> Comparison { + let starts: std::collections::HashSet = objects.iter().map(|&(u, _)| u).collect(); + let mut cmp = Comparison::default(); + for block in &valid.arena_blocks { + if block.sorted { + cmp.sorted_blocks += 1; + } else { + cmp.bitmap_blocks += 1; + } + let mut addr = block.base; + while addr < block.base + block.extent + 64 { + cmp.queries += 1; + if valid.contains(&addr) != starts.contains(&addr) { + cmp.contains_mismatches += 1; + } + let floor = objects.partition_point(|&(u, _)| u <= addr); + let expected = floor + .checked_sub(1) + .map(|i| objects[i]) + .filter(|&(u, size)| addr >= u && addr < u + size.saturating_sub(GC_HEADER_SIZE)) + .map(|(u, _)| u); + let got = valid.enclosing_object(addr); + if got != expected { + cmp.enclosing_mismatches += 1; + } + if let Some(start) = got { + cmp.enclosing_hits += 1; + if addr - start >= 64 * 8 + GC_HEADER_SIZE { + cmp.enclosing_hits_across_words += 1; + } + } + addr += 4; + } + } + cmp +} + +#[test] +fn start_bitmap_membership_and_floors_match_an_independent_arena_walk() { + run_isolated(|| { + let _planted = unsafe { plant_population() }; + let objects = oracle_objects(); + let valid = ValidPointerSetBuilder::new().finish(); + + assert_eq!( + valid.arena_count, + objects.len(), + "the census must record exactly the walkable objects" + ); + let cmp = compare_with_oracle(&valid, &objects); + assert!( + cmp.sorted_blocks >= 1, + "premise: an oversized block: {cmp:?}" + ); + assert!( + cmp.bitmap_blocks >= 2, + "premise: several bitmap blocks: {cmp:?}" + ); + assert!( + cmp.enclosing_hits_across_words > 0, + "premise: some floor search must cross bitmap words: {cmp:?}" + ); + assert_eq!(cmp.contains_mismatches, 0, "{cmp:?}"); + assert_eq!(cmp.enclosing_mismatches, 0, "{cmp:?}"); + // Index size is the point of the change: well under the 8 B/object the + // start runs cost. + assert!( + valid.arena_index_bytes() < objects.len() * 8, + "bitmap index {} B for {} objects", + valid.arena_index_bytes(), + objects.len() + ); + }); +} + +#[test] +fn sabotaged_bitmap_probe_is_caught_by_the_oracle() { + run_isolated(|| { + let _planted = unsafe { plant_population() }; + let objects = oracle_objects(); + let valid = ValidPointerSetBuilder::new().finish(); + let cmp = { + let _sabotage = start_bitmap_sabotage::Guard::arm(); + compare_with_oracle(&valid, &objects) + }; + assert!( + cmp.contains_mismatches > 0, + "a bitmap probing the neighbouring unit must disagree with the walk: {cmp:?}" + ); + }); +} + +/// A real full collection over the planted population keeps a rooted object in +/// a bitmap block and the rooted oversized object in the sorted block, and +/// reclaims a dead neighbour of the small one per object. +#[test] +fn a_full_collection_marks_through_the_bitmap_and_the_sorted_list() { + run_isolated(|| { + let planted = unsafe { plant_population() }; + let big = *planted + .iter() + .find(|&&u| { + let size = unsafe { (*header_from_user_ptr(u as *const u8)).size as usize }; + size > crate::arena::BLOCK_SIZE + }) + .expect("premise: an oversized object"); + let small = planted[4321]; + let mut small_root = string_bits(small); + let mut big_root = string_bits(big); + js_gc_register_global_root(&mut small_root as *mut u64 as i64); + js_gc_register_global_root(&mut big_root as *mut u64 as i64); + let _ = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); + for user in [small, big] { + let header = header_from_user_ptr(user as *const u8); + assert_eq!( + unsafe { (*header).obj_type }, + GC_TYPE_STRING, + "rooted object {user:#x} must survive the full" + ); + assert!(crate::arena::pointer_in_old_gen(user)); + } + assert_eq!( + unsafe { (*header_from_user_ptr(planted[4322] as *const u8)).obj_type }, + 0, + "the unrooted neighbour of the rooted small object is swept" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 0901a6549c..a24d0a07a4 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -94,42 +94,68 @@ crate::perry_thread_local! { const { std::cell::UnsafeCell::new(Vec::new()) }; } -const VALID_POINTER_ARENA_RUN_CAPACITY: usize = 1024; +/// Census blocks whose walked extent exceeds this keep a sorted start list +/// instead of a start bitmap: an oversized block (one large allocation rounded +/// up to a `BLOCK_SIZE` multiple) holds a handful of objects, and a bitmap over +/// its whole extent would be mostly zero words. +const CENSUS_BITMAP_MAX_EXTENT: usize = crate::arena::BLOCK_SIZE; + +/// Arena object starts sit on 8-byte boundaries relative to their block's +/// `data` pointer: `ArenaObjectCursor::next_budgeted` rounds every header +/// offset up to a multiple of 8 before reading it, and the census consumes +/// exactly the headers that cursor yields. +const CENSUS_START_ALIGN_SHIFT: u32 = 3; + +/// One censused arena block, in address order (#10182). +#[derive(Clone, Copy, Debug)] +pub(super) struct CensusStartBlock { + /// The block's `data` address as the census cursor snapshotted it. + pub(super) base: usize, + /// Bytes the census walked (`offset` in the snapshot). No censused header + /// starts at or past `base + extent`. + pub(super) extent: usize, + /// Global arena block index (`u32::MAX` when unknown). + pub(super) block_idx: u32, + /// Bitmap blocks: index of the block's first word in `start_bits`. + /// Sorted blocks: index of the block's first start in `large_starts`. + pub(super) first: usize, + /// Bitmap blocks: word count. Sorted blocks: start count. + pub(super) len: usize, + pub(super) sorted: bool, +} pub(crate) struct ValidPointerSet { - /// Arena-only start pointers in address-ordered runs — **the exact arena - /// membership set**, not merely an index for `enclosing_object`'s floor - /// lookups. `ArenaObjectCursorBuilder::new(ArenaWalkOrder::Address)` hands - /// the census headers over in ascending address order, so each run is - /// sorted by construction and a floor lookup that lands on the query IS - /// the membership answer. + /// **The exact arena membership set**, one entry per censused arena block + /// in ascending address order. `ArenaObjectCursorBuilder::new( + /// ArenaWalkOrder::Address)` hands the census the blocks in address order + /// and each block's headers in address order, so the table is sorted by + /// construction and a block never appears twice. + /// + /// Membership used to answer from address-ordered runs of user pointers + /// (1024 per run, sealed at block boundaries): a binary search over every + /// run's first key, then a second binary search inside the run — about 21 + /// cache-missing probes per traced pointer field, which was ~46 % of a full + /// mark on a live 20 MB JSON tree. The runs replaced a shadow `BTreeSet` + /// (#7592) and cost 8 bytes per censused object. /// - /// This used to be shadowed by a parallel `BTreeSet` over the same - /// addresses. That set cost one B-tree insert per live arena object with - /// nothing to show for it: the runs already held the same data in the same - /// order. On `json_pipeline` 500k the shadow cost **245.5 ms of a 748.3 ms - /// full collection** (`phase_us.build_valid_pointer_set`), 12.6% of the - /// `build_out` phase, and ~40 MB of transient peak heap (#7592). - pub(super) arena_runs: Vec>, - /// `arena_runs[i]`'s global arena block index (`u32::MAX` for runs pushed - /// without one). The census seals a run at every block boundary, so a - /// run never straddles two blocks and a membership hit names its block - /// for free (#10182). - pub(super) arena_run_blocks: Vec, - pub(super) current_arena_run_block: u32, + /// Now each block carries an **object-start bitmap** (1 bit per 8-byte + /// alignment unit of its walked extent, 16 KB for a full 1 MB block) and a + /// query is a search over the block fences (`arena_block_bases`, one entry + /// per block) plus one bit test. Oversized blocks keep a sorted start list + /// (`large_starts`), since they hold a few objects over many megabytes. + pub(super) arena_blocks: Vec, + /// `arena_blocks[i].base`, mirrored into one contiguous vector so the + /// block-level binary search reads 8-byte fences only. + pub(super) arena_block_bases: Vec, + /// Concatenated start bitmaps of the bitmap blocks. Bit `k` of a block's + /// bitmap is set iff a censused header starts at `base + (k << 3)`. + pub(super) start_bits: Vec, + /// Concatenated ascending start lists (user pointers) of the sorted blocks. + pub(super) large_starts: Vec, /// Per-block census facts and trace reachability (#10182). Disarmed /// unless this set was built by the production census walk. pub(super) block_census: BlockCensus, - /// `arena_runs[i].first()`, mirrored into one contiguous vector so the - /// run-level binary search reads 8-byte fences instead of chasing a - /// `Vec` header per probe. At 500k `json_pipeline` records this is ~4k - /// entries (32 KB, L2-resident) against 33 MB of run storage, and it is - /// what keeps the membership lookup competitive with the B-tree probe it - /// replaces (#7592). - pub(super) arena_run_firsts: Vec, - pub(super) current_arena_run: Vec, - /// Live count of pushed arena starts (sealed runs + the open one), kept so - /// `lookup_count` stays O(1). + /// Live count of pushed arena starts, kept so `lookup_count` stays O(1). pub(super) arena_count: usize, /// Exact membership for **malloc-tracked** objects only, which have no /// address order to exploit. A B-tree avoids hash-table rebuilds in tiny @@ -170,12 +196,11 @@ pub(crate) struct ValidPointerSet { impl ValidPointerSet { pub(super) fn new() -> Self { Self { - arena_runs: Vec::new(), - arena_run_blocks: Vec::new(), - current_arena_run_block: u32::MAX, + arena_blocks: Vec::new(), + arena_block_bases: Vec::new(), + start_bits: Vec::new(), + large_starts: Vec::new(), block_census: BlockCensus::disarmed(), - arena_run_firsts: Vec::new(), - current_arena_run: Vec::with_capacity(VALID_POINTER_ARENA_RUN_CAPACITY), arena_count: 0, malloc_lookup: std::collections::BTreeSet::new(), range_min: usize::MAX, @@ -186,34 +211,77 @@ impl ValidPointerSet { } } - /// Caller must guarantee that pushes happen in ascending address - /// order — `ValidPointerSetBuilder` does so via `ArenaObjectCursor` - /// in address order. `block_idx` is the start's global arena block; the - /// open run is sealed whenever it changes, so no run straddles two blocks - /// (#10182). - pub(super) fn push_arena_in_block(&mut self, ptr: usize, block_idx: u32) { + /// Open census block `block_idx`: `data`/`offset` exactly as the census + /// cursor snapshotted it. Every start pushed until the next call belongs to + /// this block. Blocks must be opened in ascending address order. + pub(super) fn begin_arena_block(&mut self, block_idx: u32, data: usize, offset: usize) { if self.classifier_mode { return; // #6179: no exact census in classifier mode } - if block_idx != self.current_arena_run_block { - self.seal_current_arena_run(); - self.current_arena_run_block = block_idx; - } - if let Some(previous) = self - .current_arena_run - .last() - .copied() - .or_else(|| self.arena_runs.last().and_then(|run| run.last()).copied()) - { - debug_assert!(previous <= ptr); + if let Some(previous) = self.arena_blocks.last() { + assert!( + previous.base.saturating_add(previous.extent) <= data, + "census blocks must arrive in ascending, non-overlapping address order: \ + {:#x}+{:#x} then {data:#x}", + previous.base, + previous.extent + ); } + let sorted = offset > CENSUS_BITMAP_MAX_EXTENT; + let (first, len) = if sorted { + (self.large_starts.len(), 0) + } else { + let bits = offset.div_ceil(1 << CENSUS_START_ALIGN_SHIFT); + let words = bits.div_ceil(64); + let first = self.start_bits.len(); + self.start_bits.resize(first + words, 0); + (first, words) + }; + self.arena_block_bases.push(data); + self.arena_blocks.push(CensusStartBlock { + base: data, + extent: offset, + block_idx, + first, + len, + sorted, + }); + } - self.record_pointer_range(ptr); - self.current_arena_run.push(ptr); - self.arena_count += 1; - if self.current_arena_run.len() >= VALID_POINTER_ARENA_RUN_CAPACITY { - self.seal_current_arena_run(); + /// Record a censused arena start (user pointer) in the block opened by the + /// last `begin_arena_block`. Starts arrive in ascending address order — + /// `ValidPointerSetBuilder` feeds them from `ArenaObjectCursor` in address + /// order. + pub(super) fn push_arena(&mut self, ptr: usize) { + if self.classifier_mode { + return; // #6179: no exact census in classifier mode } + let block = self + .arena_blocks + .last_mut() + .expect("an arena start is pushed only inside an opened census block"); + let header_offset = ptr.wrapping_sub(block.base).wrapping_sub(GC_HEADER_SIZE); + // A start the bitmap cannot represent would be a silent false negative + // (swept live), so the cursor's alignment contract is checked, not + // assumed. One predictable compare per censused object. + assert!( + header_offset < block.extent + && header_offset & ((1 << CENSUS_START_ALIGN_SHIFT) - 1) == 0, + "census start {ptr:#x} is outside or misaligned in its block \ + {:#x}+{:#x}", + block.base, + block.extent + ); + if block.sorted { + debug_assert!(self.large_starts.last().is_none_or(|&last| last < ptr)); + self.large_starts.push(ptr); + block.len += 1; + } else { + let bit = header_offset >> CENSUS_START_ALIGN_SHIFT; + self.start_bits[block.first + (bit >> 6)] |= 1u64 << (bit & 63); + } + self.arena_count += 1; + self.record_pointer_range(ptr); } pub(super) fn push_malloc(&mut self, ptr: usize) { @@ -236,8 +304,13 @@ impl ValidPointerSet { pub(super) fn tenured_nursery_bytes(&self) -> usize { self.tenured_nursery_bytes } - pub(super) fn finalize(&mut self) { - self.seal_current_arena_run(); + /// Transient heap bytes the arena membership index holds (fences, block + /// table, bitmaps, oversized-block start lists). + pub(super) fn arena_index_bytes(&self) -> usize { + self.arena_blocks.capacity() * std::mem::size_of::() + + self.arena_block_bases.capacity() * std::mem::size_of::() + + self.start_bits.capacity() * std::mem::size_of::() + + self.large_starts.capacity() * std::mem::size_of::() } #[inline(always)] @@ -250,18 +323,6 @@ impl ValidPointerSet { } } - fn seal_current_arena_run(&mut self) { - if self.current_arena_run.is_empty() { - return; - } - let sealed = std::mem::take(&mut self.current_arena_run); - // Non-empty by the guard above, so the fence mirror stays index-aligned - // with `arena_runs` — `arena_run_firsts[i] == arena_runs[i][0]`. - self.arena_run_firsts.push(sealed[0]); - self.arena_runs.push(sealed); - self.arena_run_blocks.push(self.current_arena_run_block); - } - /// Cheap O(1) range-rejection prefilter. Most stack words and /// register spills are not heap pointers; if the candidate falls /// outside `[range_min, range_max]` it cannot match either region @@ -292,11 +353,10 @@ impl ValidPointerSet { // censused address range. return false; } - // Exact lookup. Arena starts answer from the address-ordered census - // runs (a floor lookup that lands ON the query is membership); only - // malloc-tracked starts, which have no usable order, need the B-tree. - // Arena first because arena hits dominate every workload that reaches - // here — a malloc pointer pays one extra run-level binary search. + // Exact lookup. Arena starts answer from the per-block start bitmaps; + // only malloc-tracked starts, which have no usable order, need the + // B-tree. Arena first because arena hits dominate every workload that + // reaches here — a malloc pointer pays one extra fence search. let exact = self.arena_start_censused(*ptr) || (!self.malloc_lookup.is_empty() && self.malloc_lookup.contains(ptr)); // #6179 differential verification (PERRY_GC_VERIFY_CLASSIFIER=1): @@ -336,13 +396,14 @@ impl ValidPointerSet { /// iteration. Find the largest entry `<= query`, then validate via /// the GcHeader's size field. pub(crate) fn enclosing_object(&self, ptr: usize) -> Option { - let (candidate, run) = self.find_arena_floor_run(ptr)?; + let block = self.census_block_at(ptr)?; + let candidate = self.floor_start_in_block(block, ptr)?; unsafe { let header = (candidate as *const u8).sub(GC_HEADER_SIZE) as *const GcHeader; let total = (*header).size as usize; let payload_end = candidate + total.saturating_sub(GC_HEADER_SIZE); if ptr >= candidate && ptr < payload_end { - self.note_run_reached(run); + self.block_census.note_reached(block.block_idx); Some(candidate) } else { None @@ -350,57 +411,85 @@ impl ValidPointerSet { } } - /// A census hit in run `run`: its block was reached by the trace (#10182). - /// See `block_skip`'s module doc for why every mark passes through here. + /// The census block whose base is the greatest one `<= ptr`, if any. + /// `ptr` may still lie past that block's walked extent. #[inline(always)] - fn note_run_reached(&self, run: usize) { - if let Some(&block_idx) = self.arena_run_blocks.get(run) { - self.block_census.note_reached(block_idx); + fn census_block_at(&self, ptr: usize) -> Option<&CensusStartBlock> { + let idx = self.arena_block_bases.partition_point(|&base| base <= ptr); + if idx == 0 { + return None; } + self.arena_blocks.get(idx - 1) } - /// Exact arena membership: the census runs are address-ordered, so `ptr` - /// was censused iff its floor is itself. - /// - /// **Load-bearing ordering requirement, which the `BTreeSet` this replaced - /// did not have.** The B-tree was complete after every `push_arena`, so a - /// mid-build query merely saw fewer entries. The runs are only complete - /// once `finalize()` has sealed `current_arena_run` — up to - /// `VALID_POINTER_ARENA_RUN_CAPACITY` censused starts are invisible before - /// that. A membership query on an unsealed set is therefore a FALSE - /// NEGATIVE, and a false negative here is not a missed optimisation: the - /// conservative scan drops the root, the object is swept live, and the - /// failure surfaces cycles later as `TypeError: value is not a function`. - /// - /// The builder's phase machine guarantees this today (`Finalize` precedes - /// `Done`, and the set escapes only through `finish()`), so the assert is - /// free in release. It exists so that a phase added after `Finalize`, or a - /// caller that queries a partially-built set, fails a test instead of - /// corrupting the heap. Verified to fire: skipping the seal in `finalize` - /// trips it with "4 censused starts are invisible to this lookup". + /// Exact arena membership: a census hit also records that the trace + /// reached the hit's block (#10182; see `block_skip`'s module doc for why + /// every mark passes through here). #[inline] fn arena_start_censused(&self, ptr: usize) -> bool { - debug_assert!( - self.current_arena_run.is_empty(), - "arena membership queried before finalize() sealed the open run: \ - {} censused starts are invisible to this lookup", - self.current_arena_run.len() - ); - match self.find_arena_floor_run(ptr) { - Some((floor, run)) if floor == ptr => { - self.note_run_reached(run); - true + let Some(block) = self.census_block_at(ptr) else { + return false; + }; + let hit = if block.sorted { + self.large_starts[block.first..block.first + block.len] + .binary_search(&ptr) + .is_ok() + } else { + // Below `base + GC_HEADER_SIZE` the subtraction wraps past every + // extent, so one compare rejects both ends of the block. + let header_offset = ptr.wrapping_sub(block.base).wrapping_sub(GC_HEADER_SIZE); + if header_offset >= block.extent + || header_offset & ((1 << CENSUS_START_ALIGN_SHIFT) - 1) != 0 + { + false + } else { + let bit = header_offset >> CENSUS_START_ALIGN_SHIFT; + #[cfg(test)] + let bit = start_bitmap_sabotage::shift_probe(bit); + self.start_bits + .get(block.first + (bit >> 6)) + .is_some_and(|word| (word >> (bit & 63)) & 1 != 0) } - _ => false, + }; + if hit { + self.block_census.note_reached(block.block_idx); } + hit } - fn find_arena_floor_run(&self, ptr: usize) -> Option<(usize, usize)> { - let idx = self.arena_run_firsts.partition_point(|&first| first <= ptr); - if idx == 0 { + /// Greatest censused start (user pointer) in `block` that is `<= ptr`. + fn floor_start_in_block(&self, block: &CensusStartBlock, ptr: usize) -> Option { + if block.sorted { + let starts = &self.large_starts[block.first..block.first + block.len]; + return Self::find_floor(starts, ptr); + } + if block.extent == 0 { return None; } - Self::find_floor(&self.arena_runs[idx - 1], ptr).map(|floor| (floor, idx - 1)) + // A start `s <= ptr` has its header at `s - GC_HEADER_SIZE`, so the + // highest candidate header offset is `ptr - base - GC_HEADER_SIZE`, + // clamped to the last walked byte. + let header_offset = ptr.checked_sub(block.base)?.checked_sub(GC_HEADER_SIZE)?; + let bit = header_offset.min(block.extent - 1) >> CENSUS_START_ALIGN_SHIFT; + let mut word_idx = bit >> 6; + let top = bit & 63; + let keep = if top == 63 { + u64::MAX + } else { + (1u64 << (top + 1)) - 1 + }; + let mut word = self.start_bits[block.first + word_idx] & keep; + loop { + if word != 0 { + let floor_bit = word_idx * 64 + (63 - word.leading_zeros() as usize); + return Some(block.base + (floor_bit << CENSUS_START_ALIGN_SHIFT) + GC_HEADER_SIZE); + } + if word_idx == 0 { + return None; + } + word_idx -= 1; + word = self.start_bits[block.first + word_idx]; + } } pub(super) fn find_floor(sorted: &[usize], ptr: usize) -> Option { @@ -415,6 +504,42 @@ impl ValidPointerSet { } } +/// Sabotage switch for the start-bitmap tests: shifts every bitmap probe by +/// one alignment unit, so a test can show its membership oracle notices a +/// bitmap that answers for the wrong address. Test builds only. +#[cfg(test)] +pub(crate) mod start_bitmap_sabotage { + use std::cell::Cell; + + thread_local! { + static SHIFT_PROBE: Cell = const { Cell::new(false) }; + } + + #[inline] + pub(crate) fn shift_probe(bit: usize) -> usize { + if SHIFT_PROBE.with(Cell::get) { + bit + 1 + } else { + bit + } + } + + /// Arms the shifted probe until the guard drops. + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(SHIFT_PROBE.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + SHIFT_PROBE.with(|s| s.set(self.0)); + } + } +} + /// Build a set of all valid user-space pointers (pointers returned to callers). /// Used to validate candidates found during conservative stack scanning. pub(crate) fn build_valid_pointer_set() -> ValidPointerSet { @@ -449,8 +574,7 @@ pub(super) enum ValidPointerSetBuildPhase { pub(super) struct ValidPointerSetBuilderSnapshot { pub(super) phase: ValidPointerSetBuildPhase, pub(super) arena_setup_blocks: usize, - pub(super) arena_run_count: usize, - pub(super) current_arena_run_len: usize, + pub(super) arena_block_count: usize, pub(super) lookup_count: usize, pub(super) malloc_index: usize, } @@ -535,7 +659,6 @@ impl ValidPointerSetBuilder { return false; } self.set.block_census.flush_block(); - self.set.finalize(); self.phase = ValidPointerSetBuildPhase::Done; return true; } @@ -573,34 +696,38 @@ impl ValidPointerSetBuilder { } return false; }; - if self.census_armed { - if block_idx != self.census_block_idx { - self.census_block_idx = block_idx; - if let Some((_, data, offset)) = self - .arena_cursor - .as_ref() - .and_then(crate::arena::ArenaObjectCursor::current_block_extent) - { + if block_idx != self.census_block_idx { + self.census_block_idx = block_idx; + if let Some((_, data, offset)) = self + .arena_cursor + .as_ref() + .and_then(crate::arena::ArenaObjectCursor::current_block_extent) + { + self.set.begin_arena_block( + u32::try_from(block_idx).unwrap_or(u32::MAX), + data, + offset, + ); + if self.census_armed { self.set.block_census.begin_block(block_idx, data, offset); } } + } + if self.census_armed { unsafe { self.set .block_census .note_header(header_ptr as *const GcHeader); } } - self.record_arena_header(header_ptr, block_idx); + self.record_arena_header(header_ptr); } false } - fn record_arena_header(&mut self, header_ptr: *mut u8, block_idx: usize) { + fn record_arena_header(&mut self, header_ptr: *mut u8) { let user_ptr = unsafe { header_ptr.add(GC_HEADER_SIZE) }; - self.set.push_arena_in_block( - user_ptr as usize, - u32::try_from(block_idx).unwrap_or(u32::MAX), - ); + self.set.push_arena(user_ptr as usize); unsafe { let header = header_ptr as *const GcHeader; let flags = (*header).gc_flags; @@ -639,8 +766,7 @@ impl ValidPointerSetBuilder { .arena_cursor_builder .as_ref() .map_or(0, crate::arena::ArenaObjectCursorBuilder::inspected_blocks), - arena_run_count: self.set.arena_runs.len(), - current_arena_run_len: self.set.current_arena_run.len(), + arena_block_count: self.set.arena_blocks.len(), lookup_count: self.set.lookup_count(), malloc_index: self.malloc_index, } From 86c4e11c60346fae776dbd302b99e5c702decd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:21:32 +0200 Subject: [PATCH 02/11] gc: test-only arena index size accessor --- crates/perry-runtime/src/gc/trace.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index a24d0a07a4..f546f70c43 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -306,6 +306,7 @@ impl ValidPointerSet { } /// Transient heap bytes the arena membership index holds (fences, block /// table, bitmaps, oversized-block start lists). + #[cfg(test)] pub(super) fn arena_index_bytes(&self) -> usize { self.arena_blocks.capacity() * std::mem::size_of::() + self.arena_block_bases.capacity() * std::mem::size_of::() From 86755640bbbea83aa8292244174d3c34602ac80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:30:41 +0200 Subject: [PATCH 03/11] gc: start-bitmap test fixup --- crates/perry-runtime/src/gc/tests/start_bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/gc/tests/start_bitmap.rs b/crates/perry-runtime/src/gc/tests/start_bitmap.rs index 05c5647a41..4538848b33 100644 --- a/crates/perry-runtime/src/gc/tests/start_bitmap.rs +++ b/crates/perry-runtime/src/gc/tests/start_bitmap.rs @@ -197,7 +197,7 @@ fn a_full_collection_marks_through_the_bitmap_and_the_sorted_list() { GcTriggerKind::OldGenBytes, )); for user in [small, big] { - let header = header_from_user_ptr(user as *const u8); + let header = unsafe { header_from_user_ptr(user as *const u8) }; assert_eq!( unsafe { (*header).obj_type }, GC_TYPE_STRING, From ad777651d8fefb0780ad10005a51894a03f04518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:44:07 +0200 Subject: [PATCH 04/11] gc: allocate census start bitmaps in 8 KiB chunks --- crates/perry-runtime/src/gc/trace.rs | 84 +++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index f546f70c43..f8e75ee301 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -100,6 +100,17 @@ crate::perry_thread_local! { /// its whole extent would be mostly zero words. const CENSUS_BITMAP_MAX_EXTENT: usize = crate::arena::BLOCK_SIZE; +/// Start bitmaps are allocated in chunks of this many words (8 KiB), one or two +/// per block. A bitmap kept in one contiguous vector grew past the allocator's +/// small and medium size classes and made it commit a fresh large page: +/// `records_array_1m:sparse` read +4.8 MiB peak RSS for a ~100 KB index. The +/// start runs this replaces were 8 KiB vectors too. +const CENSUS_BITMAP_CHUNK_WORDS: usize = 1024; +const CENSUS_BITMAP_CHUNK_WORD_SHIFT: u32 = 10; +/// Chunks a bitmap block can need: `CENSUS_BITMAP_MAX_EXTENT` bytes at one bit +/// per 8 bytes is `2 * CENSUS_BITMAP_CHUNK_WORDS` words. +const CENSUS_BITMAP_MAX_CHUNKS: usize = 2; + /// Arena object starts sit on 8-byte boundaries relative to their block's /// `data` pointer: `ArenaObjectCursor::next_budgeted` rounds every header /// offset up to a multiple of 8 before reading it, and the census consumes @@ -116,7 +127,11 @@ pub(super) struct CensusStartBlock { pub(super) extent: usize, /// Global arena block index (`u32::MAX` when unknown). pub(super) block_idx: u32, - /// Bitmap blocks: index of the block's first word in `start_bits`. + /// Bitmap blocks: the block's bitmap chunks (owned by + /// `ValidPointerSet::start_bitmap_chunks`); word `w` is word + /// `w & (CENSUS_BITMAP_CHUNK_WORDS - 1)` of chunk `w >> 10`. Null past the + /// block's word count. + pub(super) chunks: [*mut u64; CENSUS_BITMAP_MAX_CHUNKS], /// Sorted blocks: index of the block's first start in `large_starts`. pub(super) first: usize, /// Bitmap blocks: word count. Sorted blocks: start count. @@ -124,6 +139,20 @@ pub(super) struct CensusStartBlock { pub(super) sorted: bool, } +impl CensusStartBlock { + /// Word `word_idx` of this bitmap block's start bitmap. + /// + /// # Safety + /// `self` is a bitmap block of a live `ValidPointerSet` and + /// `word_idx < self.len`. + #[inline(always)] + unsafe fn word_ptr(&self, word_idx: usize) -> *mut u64 { + debug_assert!(!self.sorted && word_idx < self.len); + self.chunks[word_idx >> CENSUS_BITMAP_CHUNK_WORD_SHIFT] + .add(word_idx & (CENSUS_BITMAP_CHUNK_WORDS - 1)) + } +} + pub(crate) struct ValidPointerSet { /// **The exact arena membership set**, one entry per censused arena block /// in ascending address order. `ArenaObjectCursorBuilder::new( @@ -147,9 +176,11 @@ pub(crate) struct ValidPointerSet { /// `arena_blocks[i].base`, mirrored into one contiguous vector so the /// block-level binary search reads 8-byte fences only. pub(super) arena_block_bases: Vec, - /// Concatenated start bitmaps of the bitmap blocks. Bit `k` of a block's - /// bitmap is set iff a censused header starts at `base + (k << 3)`. - pub(super) start_bits: Vec, + /// Storage of the bitmap blocks' start bitmaps, in 8 KiB chunks (see + /// `CENSUS_BITMAP_CHUNK_WORDS`). Bit `k` of a block's bitmap is set iff a + /// censused header starts at `base + (k << 3)`. A chunk's heap buffer never + /// moves, so `CensusStartBlock::chunks` may point into it. + pub(super) start_bitmap_chunks: Vec>, /// Concatenated ascending start lists (user pointers) of the sorted blocks. pub(super) large_starts: Vec, /// Per-block census facts and trace reachability (#10182). Disarmed @@ -198,7 +229,7 @@ impl ValidPointerSet { Self { arena_blocks: Vec::new(), arena_block_bases: Vec::new(), - start_bits: Vec::new(), + start_bitmap_chunks: Vec::new(), large_starts: Vec::new(), block_census: BlockCensus::disarmed(), arena_count: 0, @@ -228,20 +259,29 @@ impl ValidPointerSet { ); } let sorted = offset > CENSUS_BITMAP_MAX_EXTENT; + let mut chunks = [std::ptr::null_mut(); CENSUS_BITMAP_MAX_CHUNKS]; let (first, len) = if sorted { (self.large_starts.len(), 0) } else { let bits = offset.div_ceil(1 << CENSUS_START_ALIGN_SHIFT); let words = bits.div_ceil(64); - let first = self.start_bits.len(); - self.start_bits.resize(first + words, 0); - (first, words) + for (index, chunk) in chunks.iter_mut().enumerate() { + let start = index * CENSUS_BITMAP_CHUNK_WORDS; + if start >= words { + break; + } + let mut storage = vec![0u64; (words - start).min(CENSUS_BITMAP_CHUNK_WORDS)]; + *chunk = storage.as_mut_ptr(); + self.start_bitmap_chunks.push(storage); + } + (0, words) }; self.arena_block_bases.push(data); self.arena_blocks.push(CensusStartBlock { base: data, extent: offset, block_idx, + chunks, first, len, sorted, @@ -278,7 +318,11 @@ impl ValidPointerSet { block.len += 1; } else { let bit = header_offset >> CENSUS_START_ALIGN_SHIFT; - self.start_bits[block.first + (bit >> 6)] |= 1u64 << (bit & 63); + // SAFETY: `header_offset < extent` (asserted above), so the word + // index is below the block's word count. + unsafe { + *block.word_ptr(bit >> 6) |= 1u64 << (bit & 63); + } } self.arena_count += 1; self.record_pointer_range(ptr); @@ -310,7 +354,12 @@ impl ValidPointerSet { pub(super) fn arena_index_bytes(&self) -> usize { self.arena_blocks.capacity() * std::mem::size_of::() + self.arena_block_bases.capacity() * std::mem::size_of::() - + self.start_bits.capacity() * std::mem::size_of::() + + self + .start_bitmap_chunks + .iter() + .map(|chunk| chunk.capacity() * std::mem::size_of::()) + .sum::() + + self.start_bitmap_chunks.capacity() * std::mem::size_of::>() + self.large_starts.capacity() * std::mem::size_of::() } @@ -447,9 +496,12 @@ impl ValidPointerSet { let bit = header_offset >> CENSUS_START_ALIGN_SHIFT; #[cfg(test)] let bit = start_bitmap_sabotage::shift_probe(bit); - self.start_bits - .get(block.first + (bit >> 6)) - .is_some_and(|word| (word >> (bit & 63)) & 1 != 0) + // SAFETY: `header_offset < extent`, so the word index is below + // the block's word count (the sabotaged probe in test builds + // may step one bit past it, still inside the last word). + let word_idx = bit >> 6; + word_idx < block.len + && unsafe { (*block.word_ptr(word_idx) >> (bit & 63)) & 1 != 0 } } }; if hit { @@ -479,7 +531,9 @@ impl ValidPointerSet { } else { (1u64 << (top + 1)) - 1 }; - let mut word = self.start_bits[block.first + word_idx] & keep; + // SAFETY: `bit` derives from an offset below `extent`, and every + // later index is smaller. + let mut word = unsafe { *block.word_ptr(word_idx) } & keep; loop { if word != 0 { let floor_bit = word_idx * 64 + (63 - word.leading_zeros() as usize); @@ -489,7 +543,7 @@ impl ValidPointerSet { return None; } word_idx -= 1; - word = self.start_bits[block.first + word_idx]; + word = unsafe { *block.word_ptr(word_idx) }; } } From c3997b717fa6d3abd33b461d723b30e9c6962cd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:59:53 +0200 Subject: [PATCH 05/11] gc: skip a full's remembered-set rebuild when nothing young is marked (#10182) --- crates/perry-runtime/src/arena/mod.rs | 4 +- crates/perry-runtime/src/arena/walk.rs | 10 + crates/perry-runtime/src/gc/cycle.rs | 9 + .../src/gc/tests/full_rebuild_skip.rs | 197 ++++++++++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + crates/perry-runtime/src/gc/tests/oldgen.rs | 12 +- .../perry-runtime/src/gc/trace/block_skip.rs | 42 ++++ crates/perry-runtime/src/gc/verify.rs | 63 ++++++ 8 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 1fc2224482..35fb775743 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -102,8 +102,8 @@ pub use walk::{ pub(crate) use walk::{ arena_block_snapshots, arena_telemetry_snapshot, general_block_in_recent_window, general_block_sizes, old_arena_walk_all_headers_filtered, young_allocation_census, - ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder, ArenaTelemetrySnapshot, - ArenaWalkOrder, + young_block_count, ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder, + ArenaTelemetrySnapshot, ArenaWalkOrder, }; // reset.rs diff --git a/crates/perry-runtime/src/arena/walk.rs b/crates/perry-runtime/src/arena/walk.rs index 7f44b82f06..2031b6c9c1 100644 --- a/crates/perry-runtime/src/arena/walk.rs +++ b/crates/perry-runtime/src/arena/walk.rs @@ -905,6 +905,16 @@ pub fn general_block_count() -> usize { ARENA.with(|arena| unsafe { (*arena.get()).blocks.len() }) } +/// Global block indices `0..young_block_count()` are the young generation: +/// the general (Eden) arena's blocks, then both survivor semispaces' — the +/// order `arena_block_snapshots` and the object cursors use. +pub(crate) fn young_block_count() -> usize { + let g = ARENA.with(|arena| unsafe { (*arena.get()).blocks.len() }); + let s0 = SURVIVOR_ARENA_0.with(|arena| unsafe { (*arena.get()).blocks.len() }); + let s1 = SURVIVOR_ARENA_1.with(|arena| unsafe { (*arena.get()).blocks.len() }); + g + s0 + s1 +} + /// Per-block `size` for the general (nursery-Eden) arena, indexed by /// general block index (`0..general_block_count()`, tombstones report /// size 0). Used by the evacuation policy to translate "this block's diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 826ea115d9..2a85720828 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1294,6 +1294,15 @@ impl GcCycleState { .as_mut() .expect("atomic finalize state exists"); let rebuild = state.remembered_rebuild.get_or_insert_with(|| { + // #10182: nothing young is marked and no malloc object + // exists, so the walk could only insert nothing. + if !budgeted + && valid_ptrs.is_some_and(|ptrs| { + full_remembered_rebuild_provably_empty(&ptrs.block_census) + }) + { + return OldToYoungRememberedRebuildState::provably_empty(); + } let skip = if budgeted { None } else { diff --git a/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs b/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs new file mode 100644 index 0000000000..8ba5a7423f --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs @@ -0,0 +1,197 @@ +//! #10182: a synchronous full replaces its old→young remembered-set rebuild by +//! an exact clear when the rebuild could only produce an empty set — no marked +//! or pinned young object, and no malloc object. +//! +//! Each case runs on a fresh thread (empty arenas, empty malloc registry). The +//! protective cases plant an old→young or old→malloc edge that ONLY the rebuild +//! can recover — a raw store with no write barrier, so the pre-cycle dirty +//! snapshot does not cover it — and check the remembered set covers it after +//! the full. Their sabotaged twins force the skip and show the edge is lost. + +use super::super::*; +use super::support::*; +use crate::gc::trace::block_skip::sabotage; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + let _barriers = GeneratedWriteBarrierTestGuard::active(); + reset_global_roots(); + reset_remembered_set(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("full-rebuild-skip test thread must not panic"); +} + +fn synchronous_full() -> GcCycleTrace { + gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot { + kind: GcTriggerKind::OldGenBytes, + steps_before: Some(GcStepSnapshot::current()), + }) + .trace + .expect("full GC trace requested") +} + +/// A rooted old parent with one field. The root cell is leaked so its address +/// stays valid for the thread's lifetime. +fn rooted_old_parent() -> (usize, *mut u64) { + let (parent, fields) = unsafe { alloc_old_test_object(1) }; + let root: &'static mut u64 = Box::leak(Box::new(ptr_bits(parent as usize))); + js_gc_register_global_root(root as *mut u64 as i64); + (parent as usize, fields) +} + +/// Store `child_bits` into the parent's field WITHOUT a write barrier. +fn raw_store(parent: usize, fields: *mut u64, child_bits: u64) { + unsafe { + *fields = child_bits; + layout_note_slot(parent, 0, child_bits); + } +} + +#[test] +fn a_full_with_no_live_young_object_skips_the_rebuild() { + run_isolated(|| { + let (parent, fields) = rooted_old_parent(); + let (old_child, _) = unsafe { alloc_old_test_object(0) }; + raw_store(parent, fields, ptr_bits(old_child as usize)); + // Young garbage: the young generation is in use but nothing in it lives. + for _ in 0..512 { + let _ = unsafe { alloc_nursery_test_object(2) }; + } + let skips = crate::gc::full_remembered_rebuilds_skipped(); + + let trace = synchronous_full(); + + assert_eq!( + crate::gc::full_remembered_rebuilds_skipped(), + skips + 1, + "a full with an unmarked young generation must skip the rebuild" + ); + assert_eq!(trace.old_to_young_rebuild_objects_scanned, 0); + assert_eq!( + remembered_set_size(), + 0, + "the skipped rebuild leaves an empty set" + ); + assert_eq!(verify_old_to_young_edges_collect().missing_edges, 0); + + // The mutator's next old→young store is still remembered and survives + // a copying minor. + let child = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize; + assert!(crate::arena::pointer_in_nursery(child)); + raw_store(parent, fields, ptr_bits(child)); + js_write_barrier_slot(ptr_bits(parent), fields as u64, ptr_bits(child)); + let _ = collect_minor_trace(GcTriggerKind::Direct); + let slot_child = (unsafe { *fields } & POINTER_MASK) as usize; + assert_eq!( + unsafe { (*header_from_user_ptr(slot_child as *const u8)).obj_type }, + GC_TYPE_OBJECT, + "the barrier-recorded young child must survive the minor" + ); + }); +} + +/// The young child is reachable only through an old parent, stored without a +/// barrier: the rebuild is the only thing that can remember it. +fn plant_unbarriered_young_edge() -> (usize, *mut u64, usize) { + let (parent, fields) = rooted_old_parent(); + let child = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize; + assert!(crate::arena::pointer_in_nursery(child)); + raw_store(parent, fields, ptr_bits(child)); + assert_eq!( + remembered_set_size(), + 0, + "premise: no barrier recorded the edge" + ); + (parent, fields, child) +} + +#[test] +fn a_live_young_child_keeps_the_rebuild_and_its_edge() { + run_isolated(|| { + let (_parent, fields, child) = plant_unbarriered_young_edge(); + let skips = crate::gc::full_remembered_rebuilds_skipped(); + + let trace = synchronous_full(); + + assert_eq!(crate::gc::full_remembered_rebuilds_skipped(), skips); + assert!(trace.old_to_young_rebuild_objects_scanned > 0); + let verify = verify_old_to_young_edges_collect(); + assert!(verify.checked_old_to_young_edges > 0, "premise: {verify:?}"); + assert_eq!( + verify.missing_edges, 0, + "the rebuild must remember the edge" + ); + assert_eq!(unsafe { *fields } & POINTER_MASK, child as u64); + }); +} + +#[test] +fn sabotaged_skip_loses_an_unbarriered_young_edge() { + run_isolated(|| { + let _ = plant_unbarriered_young_edge(); + { + let _sabotage = sabotage::Guard::arm(sabotage::FORCE_REBUILD_SKIP); + let _ = synchronous_full(); + } + let verify = verify_old_to_young_edges_collect(); + assert!( + verify.missing_edges > 0, + "a wrongly skipped rebuild must leave the young edge unremembered: {verify:?}" + ); + }); +} + +/// A malloc-registry child of an old parent, stored without a barrier, with an +/// empty young generation: only the malloc guard stops the skip. +fn plant_unbarriered_malloc_edge() -> *mut u64 { + activate_malloc_registry_for_tests(); + let (parent, fields) = rooted_old_parent(); + let symbol = alloc_tracked_test_symbol() as usize; + assert!(malloc_user_ptr_tracked(symbol as *mut u8)); + raw_store(parent, fields, ptr_bits(symbol)); + assert_eq!( + remembered_set_size(), + 0, + "premise: no barrier recorded the edge" + ); + fields +} + +#[test] +fn a_live_malloc_child_keeps_the_rebuild_and_its_edge() { + run_isolated(|| { + let _fields = plant_unbarriered_malloc_edge(); + let skips = crate::gc::full_remembered_rebuilds_skipped(); + + let _ = synchronous_full(); + + assert_eq!(crate::gc::full_remembered_rebuilds_skipped(), skips); + let verify = verify_old_to_young_edges_collect(); + assert!(verify.checked_old_to_young_edges > 0, "premise: {verify:?}"); + assert_eq!( + verify.missing_edges, 0, + "the rebuild must remember the malloc edge" + ); + }); +} + +#[test] +fn sabotaged_skip_loses_an_unbarriered_malloc_edge() { + run_isolated(|| { + let _ = plant_unbarriered_malloc_edge(); + { + let _sabotage = sabotage::Guard::arm(sabotage::FORCE_REBUILD_SKIP); + let _ = synchronous_full(); + } + let verify = verify_old_to_young_edges_collect(); + assert!( + verify.missing_edges > 0, + "a wrongly skipped rebuild must leave the malloc edge unremembered: {verify:?}" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index f09c1ec962..b94cb3b4a3 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -27,6 +27,7 @@ mod forwarding_target_validation; mod forwarding_verification; mod fromspace_protect; mod fromspace_scan; +mod full_rebuild_skip; mod global_bootstrap; mod global_sink_isolation; mod handle_bound_method_name; diff --git a/crates/perry-runtime/src/gc/tests/oldgen.rs b/crates/perry-runtime/src/gc/tests/oldgen.rs index 2b26f3fc7b..c6d3bce7e4 100644 --- a/crates/perry-runtime/src/gc/tests/oldgen.rs +++ b/crates/perry-runtime/src/gc/tests/oldgen.rs @@ -1305,7 +1305,14 @@ fn test_minor_skips_whole_heap_old_to_young_rebuild() { // A full cycle DOES run the rebuild — the counter proves it is wired and // scales with the (now-large) heap, so the minor's 0 is a genuine skip - // rather than the counter being dead. + // rather than the counter being dead. #10182: a full whose young generation + // holds nothing live replaces the rebuild by an exact clear, so keep one + // pinned young object alive across it. + let (young, _young_fields) = unsafe { alloc_nursery_test_object(1) }; + let young_header = unsafe { header_from_user_ptr(young as *const u8) }; + unsafe { + crate::gc::pin_object(young_header); + } let full_outcome = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot { kind: GcTriggerKind::Direct, steps_before: Some(GcStepSnapshot::current()), @@ -1316,6 +1323,9 @@ fn test_minor_skips_whole_heap_old_to_young_rebuild() { "a full cycle must walk the whole heap for the RS rebuild (got {}, expected >= {OLD_OBJECTS})", full_trace.old_to_young_rebuild_objects_scanned, ); + unsafe { + crate::gc::unpin_object(young_header); + } for header in old_headers { unsafe { diff --git a/crates/perry-runtime/src/gc/trace/block_skip.rs b/crates/perry-runtime/src/gc/trace/block_skip.rs index a04e9a3b04..22d53fb18b 100644 --- a/crates/perry-runtime/src/gc/trace/block_skip.rs +++ b/crates/perry-runtime/src/gc/trace/block_skip.rs @@ -62,6 +62,10 @@ pub(crate) struct CensusBlock { pub(crate) censused: bool, /// Some object in the block needs the per-object sweep path. pub(crate) obligation: bool, + /// Some header in the block was already MARKED or PINNED when the census + /// read it (a subset of `obligation`, kept apart for + /// `young_generation_unmarked`). + pub(crate) premarked: bool, } /// Per-block census facts and trace reachability for one cycle's @@ -135,6 +139,7 @@ impl BlockCensus { bytes: 0, censused: true, obligation: false, + premarked: false, }; } @@ -151,6 +156,7 @@ impl BlockCensus { let exceptional_flags = (flags ^ GC_FLAG_ARENA) & (GC_FLAG_ARENA | GC_FLAG_MARKED | GC_FLAG_PINNED | GC_FLAG_FORWARDED) != 0; + let premarked = flags & (GC_FLAG_MARKED | GC_FLAG_PINNED) != 0; let raw_f64_array = obj_type == GC_TYPE_ARRAY && (*header)._reserved & (GC_ARRAY_RAW_F64_LAYOUT | GC_ARRAY_RAW_F64_HOLES) != 0; let type_obligation = self.obligation_by_type[obj_type as usize]; @@ -158,6 +164,7 @@ impl BlockCensus { block.objects += 1; block.bytes += size; block.obligation |= exceptional_flags | type_obligation | raw_f64_array; + block.premarked |= premarked; } /// Fold the current block into the per-index table. Called at every block @@ -213,6 +220,38 @@ impl BlockCensus { any.then_some(skip) } + /// After the mark of a synchronous full: does the young generation (Eden + /// and both survivor spaces) hold **no** marked or pinned object? + /// + /// Every in-use young block must be censused, unchanged since the census + /// (no allocate-black birth, no block created after it), free of headers + /// that were already marked or pinned when censused, and unreached by the + /// trace. An unreached block holds no object the trace marked (see this + /// module's doc: every census-built mark passes a membership query that + /// records its block), so every young object is then garbage. `false` when + /// the census is disarmed or anything is uncertain. + pub(crate) fn young_generation_unmarked(&self) -> bool { + if !self.armed { + return false; + } + let snapshots = crate::arena::arena_block_snapshots(); + let young = crate::arena::young_block_count().min(snapshots.len()); + snapshots[..young] + .iter() + .enumerate() + .all(|(block_idx, snapshot)| { + if snapshot.data == 0 || snapshot.offset == 0 { + return true; + } + self.block(block_idx).is_some_and(|block| { + !block.premarked + && !self.reached(block_idx) + && block.data == snapshot.data + && block.end == snapshot.data.saturating_add(snapshot.offset) + }) + }) + } + pub(crate) fn block(&self, block_idx: usize) -> Option { self.blocks.get(block_idx).copied().filter(|b| b.censused) } @@ -231,6 +270,9 @@ pub(crate) mod sabotage { pub(crate) const FORGET_REACHED: u8 = 1; pub(crate) const FORGET_OBLIGATIONS: u8 = 2; + /// `verify::full_remembered_rebuild_provably_empty` answers true whatever + /// the heap holds. + pub(crate) const FORCE_REBUILD_SKIP: u8 = 4; thread_local! { static SABOTAGE: Cell = const { Cell::new(0) }; diff --git a/crates/perry-runtime/src/gc/verify.rs b/crates/perry-runtime/src/gc/verify.rs index 36e6038046..48a642a65d 100644 --- a/crates/perry-runtime/src/gc/verify.rs +++ b/crates/perry-runtime/src/gc/verify.rs @@ -550,6 +550,48 @@ unsafe fn remember_retained_old_to_young_slots( }); } +crate::perry_thread_local! { + /// Synchronous full collections whose old→young remembered-set rebuild was + /// replaced by an exact clear because the young generation held no marked + /// object (#10182). Live-subject counter for the tests and the diag line. + static FULL_REMEMBERED_REBUILDS_SKIPPED: std::cell::Cell = + const { std::cell::Cell::new(0) }; +} + +/// Running count of [`FULL_REMEMBERED_REBUILDS_SKIPPED`] on this thread. +pub(crate) fn full_remembered_rebuilds_skipped() -> u64 { + FULL_REMEMBERED_REBUILDS_SKIPPED.with(std::cell::Cell::get) +} + +/// #10182: after a synchronous full's mark, can the old→young remembered-set +/// rebuild only produce an empty set? +/// +/// The rebuild remembers a slot of a marked (or pinned) old parent exactly when +/// its child classifies as young (nursery) or is a registered malloc object +/// (`remembered_child_needs_tracking`). A marked parent's strong child is +/// marked too, and the rebuild skips weak slots exactly as the trace does. So: +/// +/// * if no young object is marked or pinned (`young_generation_unmarked`), +/// every young child a marked parent could name is garbage the sweep is +/// about to reclaim, and no mutator runs in between to make one live; and +/// * if the malloc registry is empty — the same premise the copying minor's +/// `skip_remembering` uses — there is no malloc child at all, +/// +/// then the walk can insert nothing that names a live object, and the +/// remembered set this full leaves behind is exactly empty. The pre-cycle +/// dirty snapshot repair (`restore_surviving_dirty_coverage`) still runs in +/// reclaim as before. A budgeted cycle has no census, so it never qualifies. +pub(super) fn full_remembered_rebuild_provably_empty(census: &super::trace::BlockCensus) -> bool { + #[cfg(test)] + if super::trace::block_skip::sabotage::get() + & super::trace::block_skip::sabotage::FORCE_REBUILD_SKIP + != 0 + { + return census.is_armed(); + } + census.young_generation_unmarked() && MALLOC_STATE.with(|s| s.borrow().objects.is_empty()) +} + pub(super) struct OldToYoungRememberedRebuildState { require_marked: bool, sticky: StickyRememberedSet, @@ -586,6 +628,27 @@ impl OldToYoungRememberedRebuildState { } } + /// The rebuild of a full whose result is provably empty + /// (`full_remembered_rebuild_provably_empty`): no walk, an empty set. + pub(super) fn provably_empty() -> Self { + FULL_REMEMBERED_REBUILDS_SKIPPED.with(|c| c.set(c.get().saturating_add(1))); + if crate::gc::gc_diag_enabled() { + eprintln!( + "[gc-remembered-rebuild] full skipped=young_generation_unmarked skips_total={}", + full_remembered_rebuilds_skipped() + ); + } + Self { + require_marked: true, + sticky: StickyRememberedSet::default(), + arena_cursor: None, + arena_done: true, + malloc_index: 0, + objects_scanned: 0, + done: true, + } + } + /// Number of heap objects this whole-heap rebuild walk has visited. Used /// by the GC trace to prove that minors do NOT run this O(all-objects) /// walk (#6181): full cycles report the walked object count, minors 0. From 104efdcabcaef49f896989c7ca3e5fdc953a2fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 20:11:23 +0200 Subject: [PATCH 06/11] gc: full-rebuild-skip tests check the known-live parent --- .../src/gc/tests/full_rebuild_skip.rs | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs b/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs index 8ba5a7423f..8675d1600b 100644 --- a/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs +++ b/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs @@ -44,6 +44,21 @@ fn rooted_old_parent() -> (usize, *mut u64) { (parent as usize, fields) } +/// The old→young edge verifier checks a parent only while it is marked or +/// pinned (or already remembered); after a completed full no mark is left, so +/// check the known-live parent explicitly. +fn verify_live_parent(parent: usize) -> OldYoungEdgeVerifyStats { + let header = unsafe { header_from_user_ptr(parent as *const u8) }; + unsafe { + (*header).gc_flags |= GC_FLAG_MARKED; + } + let stats = verify_old_to_young_edges_collect(); + unsafe { + (*header).gc_flags &= !GC_FLAG_MARKED; + } + stats +} + /// Store `child_bits` into the parent's field WITHOUT a write barrier. fn raw_store(parent: usize, fields: *mut u64, child_bits: u64) { unsafe { @@ -77,7 +92,8 @@ fn a_full_with_no_live_young_object_skips_the_rebuild() { 0, "the skipped rebuild leaves an empty set" ); - assert_eq!(verify_old_to_young_edges_collect().missing_edges, 0); + let verify = verify_live_parent(parent); + assert_eq!(verify.missing_edges, 0, "{verify:?}"); // The mutator's next old→young store is still remembered and survives // a copying minor. @@ -113,14 +129,14 @@ fn plant_unbarriered_young_edge() -> (usize, *mut u64, usize) { #[test] fn a_live_young_child_keeps_the_rebuild_and_its_edge() { run_isolated(|| { - let (_parent, fields, child) = plant_unbarriered_young_edge(); + let (parent, fields, child) = plant_unbarriered_young_edge(); let skips = crate::gc::full_remembered_rebuilds_skipped(); let trace = synchronous_full(); assert_eq!(crate::gc::full_remembered_rebuilds_skipped(), skips); assert!(trace.old_to_young_rebuild_objects_scanned > 0); - let verify = verify_old_to_young_edges_collect(); + let verify = verify_live_parent(parent); assert!(verify.checked_old_to_young_edges > 0, "premise: {verify:?}"); assert_eq!( verify.missing_edges, 0, @@ -133,12 +149,12 @@ fn a_live_young_child_keeps_the_rebuild_and_its_edge() { #[test] fn sabotaged_skip_loses_an_unbarriered_young_edge() { run_isolated(|| { - let _ = plant_unbarriered_young_edge(); + let (parent, _, _) = plant_unbarriered_young_edge(); { let _sabotage = sabotage::Guard::arm(sabotage::FORCE_REBUILD_SKIP); let _ = synchronous_full(); } - let verify = verify_old_to_young_edges_collect(); + let verify = verify_live_parent(parent); assert!( verify.missing_edges > 0, "a wrongly skipped rebuild must leave the young edge unremembered: {verify:?}" @@ -148,7 +164,7 @@ fn sabotaged_skip_loses_an_unbarriered_young_edge() { /// A malloc-registry child of an old parent, stored without a barrier, with an /// empty young generation: only the malloc guard stops the skip. -fn plant_unbarriered_malloc_edge() -> *mut u64 { +fn plant_unbarriered_malloc_edge() -> usize { activate_malloc_registry_for_tests(); let (parent, fields) = rooted_old_parent(); let symbol = alloc_tracked_test_symbol() as usize; @@ -159,19 +175,19 @@ fn plant_unbarriered_malloc_edge() -> *mut u64 { 0, "premise: no barrier recorded the edge" ); - fields + parent } #[test] fn a_live_malloc_child_keeps_the_rebuild_and_its_edge() { run_isolated(|| { - let _fields = plant_unbarriered_malloc_edge(); + let parent = plant_unbarriered_malloc_edge(); let skips = crate::gc::full_remembered_rebuilds_skipped(); let _ = synchronous_full(); assert_eq!(crate::gc::full_remembered_rebuilds_skipped(), skips); - let verify = verify_old_to_young_edges_collect(); + let verify = verify_live_parent(parent); assert!(verify.checked_old_to_young_edges > 0, "premise: {verify:?}"); assert_eq!( verify.missing_edges, 0, @@ -183,12 +199,12 @@ fn a_live_malloc_child_keeps_the_rebuild_and_its_edge() { #[test] fn sabotaged_skip_loses_an_unbarriered_malloc_edge() { run_isolated(|| { - let _ = plant_unbarriered_malloc_edge(); + let parent = plant_unbarriered_malloc_edge(); { let _sabotage = sabotage::Guard::arm(sabotage::FORCE_REBUILD_SKIP); let _ = synchronous_full(); } - let verify = verify_old_to_young_edges_collect(); + let verify = verify_live_parent(parent); assert!( verify.missing_edges > 0, "a wrongly skipped rebuild must leave the malloc edge unremembered: {verify:?}" From 7101bea99c8dc90b5533fa755128402a93786333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 20:24:02 +0200 Subject: [PATCH 07/11] gc: apply a full sweep's old-page accounting once per page (#10182) --- crates/perry-runtime/src/arena/mod.rs | 13 +- .../perry-runtime/src/arena/page_meta/mod.rs | 4 + .../src/arena/page_meta/sweep_tally.rs | 87 ++++++ .../src/gc/oldgen/sweep_batch.rs | 7 + .../src/gc/oldgen/sweep_objects.rs | 93 +++++- crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/sweep_page_tally.rs | 265 ++++++++++++++++++ .../perry-runtime/src/gc/trace/block_skip.rs | 3 + 8 files changed, 455 insertions(+), 18 deletions(-) create mode 100644 crates/perry-runtime/src/arena/page_meta/sweep_tally.rs create mode 100644 crates/perry-runtime/src/gc/tests/sweep_page_tally.rs diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 35fb775743..fa5c307b93 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -150,12 +150,13 @@ pub(crate) use page_meta::{ classify_heap_space_in_range, generation_page_for_addr, materialize_all_promoted_page_runs, old_arena_block_range_index, old_arena_block_ranges, old_arena_page_index_remove_object, old_arena_source_blocks_for_pages, old_arena_walk_objects_on_pages, old_object_page_overlaps, - old_page_account_dirty_slot, old_page_account_dirty_slots, old_page_account_promoted_object, - old_page_account_swept_object, old_page_clear_dirty, old_page_mark_dirty, - old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle, - old_pages_reset_sweep_accounting, record_arena_object_start, unregister_old_object_pages, - unregister_old_objects_batch, HeapGeneration, HeapSpace, OldArenaPageObjectCursor, - OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, + old_object_single_page, old_page_account_dirty_slot, old_page_account_dirty_slots, + old_page_account_promoted_object, old_page_account_swept_object, old_page_account_swept_tally, + old_page_clear_dirty, old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, + old_pages_begin_gc_cycle, old_pages_reset_sweep_accounting, record_arena_object_start, + unregister_old_object_pages, unregister_old_objects_batch, HeapGeneration, HeapSpace, + OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, + OldPageSweepTally, }; #[cfg(test)] diff --git a/crates/perry-runtime/src/arena/page_meta/mod.rs b/crates/perry-runtime/src/arena/page_meta/mod.rs index f2f0b180b6..7575141fac 100644 --- a/crates/perry-runtime/src/arena/page_meta/mod.rs +++ b/crates/perry-runtime/src/arena/page_meta/mod.rs @@ -139,7 +139,11 @@ impl PageGenerationCache { const PAGE_GENERATION_CACHE_WAYS: usize = 4; mod page_class; +mod sweep_tally; pub(crate) use page_class::*; +pub(crate) use sweep_tally::{ + old_object_single_page, old_page_account_swept_tally, OldPageSweepTally, +}; #[cfg(test)] mod tests; diff --git a/crates/perry-runtime/src/arena/page_meta/sweep_tally.rs b/crates/perry-runtime/src/arena/page_meta/sweep_tally.rs new file mode 100644 index 0000000000..1a43e2488c --- /dev/null +++ b/crates/perry-runtime/src/arena/page_meta/sweep_tally.rs @@ -0,0 +1,87 @@ +//! Per-page batching of a sweep's old-generation page accounting (#10182). +//! +//! `old_page_account_swept_object` costs one heap-allocated overlap vector and +//! one page-meta hash lookup per swept old object. A full sweep over a live +//! 20 MB JSON tree calls it ~585k times, and it was about three quarters of the +//! sweep. The sweep walks each block in address order, so consecutive objects +//! share a page: summing their deltas and applying the sum once per page gives +//! the same page meta, because every field involved is a plain sum and +//! `refresh_policy_bits` is a pure recompute of the page's own fields. +//! +//! The one interleaving that could observe the difference is the batched +//! page-index removal (`unregister_old_objects_batch`), which zeroes a page's +//! sweep accounting when its last object leaves. The sweep therefore applies +//! its tally before every such flush, which keeps the order of resets and sums +//! on every page exactly as the one-by-one calls had it. + +use super::*; + +/// Sweep accounting deltas of consecutive old objects that each lie wholly on +/// one page. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct OldPageSweepTally { + pub(crate) live_bytes: usize, + pub(crate) live_objects: usize, + pub(crate) pinned_bytes: usize, + pub(crate) pinned_objects: usize, + pub(crate) dead_bytes: usize, + pub(crate) dead_objects: usize, +} + +impl OldPageSweepTally { + #[inline(always)] + pub(crate) fn add(&mut self, total_size: usize, live: bool, pinned: bool) { + if live { + self.live_bytes += total_size; + self.live_objects += 1; + if pinned { + self.pinned_bytes += total_size; + self.pinned_objects += 1; + } + } else { + self.dead_bytes += total_size; + self.dead_objects += 1; + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.live_objects == 0 && self.dead_objects == 0 + } +} + +/// The page an old object lies on when it lies wholly on one page. +#[inline(always)] +pub(crate) fn old_object_single_page(header_addr: usize, total_size: usize) -> Option { + if header_addr == 0 || total_size == 0 { + return None; + } + let first_page = generation_page_for_addr(header_addr); + (first_page == generation_page_for_addr(header_addr + total_size - 1)).then_some(first_page) +} + +/// Apply `tally` to `page` exactly as the same objects' one-by-one +/// `old_page_account_swept_object` calls would have. +pub(crate) fn old_page_account_swept_tally(page: usize, tally: &OldPageSweepTally) { + if tally.is_empty() { + return; + } + OLD_GEN_PAGE_META.with(|meta| { + let mut meta = meta.borrow_mut(); + let page_meta = meta + .entry(page) + .or_insert_with(|| OldPageMeta::zero_for_page(page)); + page_meta.live_bytes = page_meta.live_bytes.saturating_add(tally.live_bytes); + page_meta.live_object_count = page_meta + .live_object_count + .saturating_add(tally.live_objects); + page_meta.pinned_bytes = page_meta.pinned_bytes.saturating_add(tally.pinned_bytes); + page_meta.pinned_object_count = page_meta + .pinned_object_count + .saturating_add(tally.pinned_objects); + page_meta.dead_bytes = page_meta.dead_bytes.saturating_add(tally.dead_bytes); + page_meta.dead_object_count = page_meta + .dead_object_count + .saturating_add(tally.dead_objects); + page_meta.refresh_policy_bits(); + }); +} diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs index db13089b1c..74daed90c1 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs @@ -35,6 +35,13 @@ impl PendingOldUnregister { } } + /// Will the next `defer` flush the queue (and so zero the sweep accounting + /// of every page whose last object it removes)? + #[inline] + pub(super) fn flushes_on_next_defer(&self) -> bool { + self.dead.len() + 1 >= FLUSH_AT + } + /// Remove every queued header from the page index. pub(super) fn flush(&mut self) { if self.dead.is_empty() { diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs index b488837772..0cd37bc9ab 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs @@ -7,6 +7,10 @@ pub(super) struct ArenaSweepObjectsState { cursor: crate::arena::ArenaObjectCursor, /// Dead old headers awaiting one batched page-index removal (see `sweep_batch`). pending_old_unregister: super::sweep_batch::PendingOldUnregister, + /// Page accounting of consecutive single-page old objects, applied once per + /// page (see `arena::page_meta::sweep_tally`). `usize::MAX` when empty. + old_page_tally_page: usize, + old_page_tally: crate::arena::OldPageSweepTally, block_snapshots: Vec, block_has_live: Vec, resettable_general_n: usize, @@ -71,6 +75,8 @@ impl ArenaSweepObjectsState { Self { cursor: crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::BlockIndex), pending_old_unregister: Default::default(), + old_page_tally_page: usize::MAX, + old_page_tally: Default::default(), block_snapshots, block_has_live: vec![false; n_blocks], resettable_general_n: crate::arena::general_block_count(), @@ -218,11 +224,79 @@ impl ArenaSweepObjectsState { remaining -= 1; self.process_object(header_ptr as *mut GcHeader, block_idx); } - // Never leave a dead header in the page index across a step boundary. - self.pending_old_unregister.flush(); + // Never leave a dead header in the page index, or a page's accounting + // unapplied, across a step boundary. The tally goes first: the + // unregister flush zeroes the accounting of pages it empties. + if self.page_tally_order_kept() { + self.apply_old_page_tally(); + self.pending_old_unregister.flush(); + } else { + self.pending_old_unregister.flush(); + self.apply_old_page_tally(); + } done } + /// Account one swept old object on its page(s), batching single-page + /// objects per page. + #[inline] + fn account_old_object( + &mut self, + header: *mut GcHeader, + total_size: usize, + live: bool, + pinned: bool, + ) { + match crate::arena::old_object_single_page(header as usize, total_size) { + Some(page) => { + if page != self.old_page_tally_page { + self.apply_old_page_tally(); + self.old_page_tally_page = page; + } + self.old_page_tally.add(total_size, live, pinned); + } + None => crate::arena::old_page_account_swept_object( + header as usize, + total_size, + live, + pinned, + ), + } + } + + fn apply_old_page_tally(&mut self) { + if self.old_page_tally_page != usize::MAX { + crate::arena::old_page_account_swept_tally( + self.old_page_tally_page, + &self.old_page_tally, + ); + } + self.old_page_tally_page = usize::MAX; + self.old_page_tally = Default::default(); + } + + /// The tally is applied before every page-index flush (always, outside the + /// sabotaged test). + #[inline(always)] + fn page_tally_order_kept(&self) -> bool { + #[cfg(test)] + { + use super::super::trace::block_skip::sabotage; + sabotage::get() & sabotage::FORGET_PAGE_TALLY_ORDER == 0 + } + #[cfg(not(test))] + true + } + + /// Queue a dead old header's page-index removal, applying the page tally + /// first when the queue is about to flush. + unsafe fn defer_old_unregister(&mut self, header: *mut GcHeader, total_size: usize) { + if self.pending_old_unregister.flushes_on_next_defer() && self.page_tally_order_kept() { + self.apply_old_page_tally(); + } + self.pending_old_unregister.defer(header, total_size); + } + pub(super) fn block_has_live(&self) -> &[bool] { &self.block_has_live } @@ -315,12 +389,7 @@ impl ArenaSweepObjectsState { count_in_live_census: bool, ) { if block_idx >= self.old_block_start { - crate::arena::old_page_account_swept_object( - header as usize, - (*header).size as usize, - true, - pinned, - ); + self.account_old_object(header, (*header).size as usize, true, pinned); } if block_idx < self.block_has_live.len() { self.block_has_live[block_idx] = true; @@ -386,7 +455,7 @@ impl ArenaSweepObjectsState { let total_size = (*header).size as usize; let dead_old = block_idx >= self.old_block_start; if dead_old { - crate::arena::old_page_account_swept_object(header as usize, total_size, false, false); + self.account_old_object(header, total_size, false, false); } let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE); self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64); @@ -395,7 +464,7 @@ impl ArenaSweepObjectsState { gc_type_clear_dead_payload_side_tables((*header).obj_type, user_ptr as usize); } if self.reclaim_dead_old_blocks && dead_old { - self.pending_old_unregister.defer(header, total_size); + self.defer_old_unregister(header, total_size); } else { (*header).gc_flags = flags & !(GC_FLAG_FORWARDED | GC_FLAG_MARKED); } @@ -405,7 +474,7 @@ impl ArenaSweepObjectsState { let total_size = (*header).size as usize; let dead_old = block_idx >= self.old_block_start; if dead_old { - crate::arena::old_page_account_swept_object(header as usize, total_size, false, false); + self.account_old_object(header, total_size, false, false); } let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE); self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64); @@ -414,7 +483,7 @@ impl ArenaSweepObjectsState { } finalize_dead_arena_payload(header, user_ptr, self.overflow_active); if self.reclaim_dead_old_blocks && dead_old { - self.pending_old_unregister.defer(header, total_size); + self.defer_old_unregister(header, total_size); } } } diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index b94cb3b4a3..bb3696fe35 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -69,6 +69,7 @@ mod start_bitmap; mod step_bounds; pub(super) mod support; mod survival_diag; +mod sweep_page_tally; mod teardown; mod telemetry_verifier; mod temp_roots; diff --git a/crates/perry-runtime/src/gc/tests/sweep_page_tally.rs b/crates/perry-runtime/src/gc/tests/sweep_page_tally.rs new file mode 100644 index 0000000000..c93310bcd1 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/sweep_page_tally.rs @@ -0,0 +1,265 @@ +//! #10182: a full sweep applies old-generation page accounting once per page +//! (`arena::page_meta::sweep_tally`) instead of once per object. +//! +//! The case plants an old population whose liveness is known object by object +//! — live, pinned and dead objects of assorted sizes, objects spanning pages, +//! whole pages of dead objects inside live blocks, and more dead objects than +//! one page-index flush holds — runs one synchronous full, and compares every +//! planted page's accounting with an oracle computed from the plan: a page +//! any surviving object overlaps sums its live, pinned and dead overlaps; a +//! page no survivor overlaps is emptied by the page-index removal, which zeroes +//! its accounting. The sabotaged twin applies the tally after the page-index +//! flush instead of before and shows the comparison notices. + +use super::super::*; +use super::support::*; +use crate::gc::trace::block_skip::sabotage; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("sweep page-tally test thread must not panic"); +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Fate { + Live, + Pinned, + Dead, +} + +struct Planted { + header: usize, + size: usize, + fate: Fate, +} + +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +struct PageExpect { + live_bytes: usize, + live_objects: usize, + pinned_bytes: usize, + pinned_objects: usize, + dead_bytes: usize, + dead_objects: usize, +} + +const PAGE: usize = crate::arena::GENERATION_PAGE_SIZE; + +/// Plant the population. Live objects are held by one rooted old array; pinned +/// objects are pinned. The plan ends with a run of dead objects covering whole +/// pages, so the last page the sweep visits holds no survivor. +unsafe fn plant() -> Vec { + const LIVE_SLOTS: u32 = 2048; + let (holder, elements) = alloc_old_test_array(LIVE_SLOTS); + let root: &'static mut u64 = Box::leak(Box::new(ptr_bits(holder as usize))); + js_gc_register_global_root(root as *mut u64 as i64); + let mut planted = vec![Planted { + header: holder as usize - GC_HEADER_SIZE, + size: old_test_header_and_size(holder as usize).1, + fate: Fate::Live, + }]; + let mut live_slot = 0u32; + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let mut next = || { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (state >> 33) as usize + }; + let mut allocated = 0usize; + while allocated < 3 * crate::arena::BLOCK_SIZE { + let roll = next(); + let payload = if roll % 29 == 0 { + 5000 + roll % 7000 + } else { + 8 + (roll >> 5) % 600 + }; + let fate = match (roll >> 12) % 64 { + 0 => Fate::Pinned, + 1..=7 if live_slot < LIVE_SLOTS => Fate::Live, + _ => Fate::Dead, + }; + let user = crate::arena::arena_alloc_gc_old(payload, 8, GC_TYPE_STRING) as usize; + let size = old_test_header_and_size(user).1; + match fate { + Fate::Live => { + *elements.add(live_slot as usize) = string_bits(user); + live_slot += 1; + } + Fate::Pinned => crate::gc::pin_object(header_from_user_ptr(user as *const u8)), + Fate::Dead => {} + } + planted.push(Planted { + header: user - GC_HEADER_SIZE, + size, + fate, + }); + allocated += size; + // Every so often, a run of dead objects covering whole pages. + if roll % 97 == 0 { + let mut run = 0; + while run < 3 * PAGE { + let user = crate::arena::arena_alloc_gc_old(120, 8, GC_TYPE_STRING) as usize; + let size = old_test_header_and_size(user).1; + planted.push(Planted { + header: user - GC_HEADER_SIZE, + size, + fate: Fate::Dead, + }); + run += size; + allocated += size; + } + } + } + let mut run = 0; + while run < 3 * PAGE { + let user = crate::arena::arena_alloc_gc_old(120, 8, GC_TYPE_STRING) as usize; + let size = old_test_header_and_size(user).1; + planted.push(Planted { + header: user - GC_HEADER_SIZE, + size, + fate: Fate::Dead, + }); + run += size; + } + planted +} + +fn oracle(planted: &[Planted]) -> std::collections::BTreeMap { + let mut pages: std::collections::BTreeMap = Default::default(); + for object in planted { + let end = object.header + object.size; + let mut page_base = object.header & !(PAGE - 1); + while page_base < end { + let overlap = end.min(page_base + PAGE) - object.header.max(page_base); + let (expect, survivor) = pages.entry(page_base).or_default(); + match object.fate { + Fate::Live | Fate::Pinned => { + *survivor = true; + expect.live_bytes += overlap; + expect.live_objects += 1; + if object.fate == Fate::Pinned { + expect.pinned_bytes += overlap; + expect.pinned_objects += 1; + } + } + Fate::Dead => { + expect.dead_bytes += overlap; + expect.dead_objects += 1; + } + } + page_base += PAGE; + } + } + pages +} + +#[derive(Debug, Default)] +struct Comparison { + pages: usize, + survivor_pages_with_dead: usize, + emptied_pages: usize, + multi_page_objects: usize, + dead_objects: usize, + mismatches: usize, + first_mismatch: Option<(usize, PageExpect, PageExpect)>, +} + +fn compare(planted: &[Planted]) -> Comparison { + let meta: std::collections::HashMap = + crate::arena::old_page_meta_snapshot() + .into_iter() + .map(|m| (m.page_base, m)) + .collect(); + let mut cmp = Comparison { + multi_page_objects: planted + .iter() + .filter(|o| o.header / PAGE != (o.header + o.size - 1) / PAGE) + .count(), + dead_objects: planted.iter().filter(|o| o.fate == Fate::Dead).count(), + ..Comparison::default() + }; + for (page_base, (expect, survivor)) in oracle(planted) { + cmp.pages += 1; + let expect = if survivor { + if expect.dead_objects > 0 { + cmp.survivor_pages_with_dead += 1; + } + expect + } else { + cmp.emptied_pages += 1; + PageExpect::default() + }; + let got = meta + .get(&page_base) + .map_or(PageExpect::default(), |m| PageExpect { + live_bytes: m.live_bytes, + live_objects: m.live_object_count, + pinned_bytes: m.pinned_bytes, + pinned_objects: m.pinned_object_count, + dead_bytes: m.dead_bytes, + dead_objects: m.dead_object_count, + }); + let eligible_ok = meta.get(&page_base).is_none_or(|m| { + m.evacuation_eligible + == (m.allocated_bytes > 0 + && m.live_bytes > 0 + && m.dead_bytes > 0 + && m.pinned_bytes == 0) + }); + if got != expect || !eligible_ok { + cmp.mismatches += 1; + cmp.first_mismatch.get_or_insert((page_base, expect, got)); + } + } + cmp +} + +fn full() { + let _ = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); +} + +#[test] +fn full_sweep_page_accounting_matches_the_planted_liveness() { + run_isolated(|| { + let planted = unsafe { plant() }; + full(); + let cmp = compare(&planted); + assert!( + cmp.emptied_pages >= 3, + "premise: pages with no survivor: {cmp:?}" + ); + assert!(cmp.survivor_pages_with_dead >= 100, "premise: {cmp:?}"); + assert!(cmp.multi_page_objects >= 10, "premise: {cmp:?}"); + assert!( + cmp.dead_objects > 4096, + "premise: more than one page-index flush: {cmp:?}" + ); + assert_eq!(cmp.mismatches, 0, "{cmp:?}"); + }); +} + +#[test] +fn sabotaged_tally_order_is_caught_by_the_oracle() { + run_isolated(|| { + let planted = unsafe { plant() }; + { + let _sabotage = sabotage::Guard::arm(sabotage::FORGET_PAGE_TALLY_ORDER); + full(); + } + let cmp = compare(&planted); + assert!( + cmp.mismatches > 0, + "a tally applied after the page-index flush must leave an emptied page's accounting: {cmp:?}" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/trace/block_skip.rs b/crates/perry-runtime/src/gc/trace/block_skip.rs index 22d53fb18b..dee36a19bd 100644 --- a/crates/perry-runtime/src/gc/trace/block_skip.rs +++ b/crates/perry-runtime/src/gc/trace/block_skip.rs @@ -273,6 +273,9 @@ pub(crate) mod sabotage { /// `verify::full_remembered_rebuild_provably_empty` answers true whatever /// the heap holds. pub(crate) const FORCE_REBUILD_SKIP: u8 = 4; + /// The sweep never applies its per-page accounting tally before a page-index + /// flush or a step end (it is applied only at page changes). + pub(crate) const FORGET_PAGE_TALLY_ORDER: u8 = 8; thread_local! { static SABOTAGE: Cell = const { Cell::new(0) }; From 9b2b7a57a0ef5bf51dd8e5d8ad7e6cd8854205b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 21:22:49 +0200 Subject: [PATCH 08/11] gc: classify the full-rebuild skip counter and re-pin the census window --- scripts/gc_runtime_root_holders.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 533ae24984..f69703e576 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -307,7 +307,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -323,7 +323,7 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "3f9e6be47b4454022b70ff2357bbdf3a80ef6a84c4986e58763acbdcce9142c1", - "crates/perry-runtime/src/gc/cycle.rs": "fdef083301463ad8cec8142ca86cc4354e289c67e97bbc6caea2e8d16d4bf089", + "crates/perry-runtime/src/gc/cycle.rs": "b65e82014de18a1746fe9563f0c5bcef202f1a66e7497d51b4556b04c913db6e", "crates/perry-runtime/src/gc/mod.rs": "90339683735e4d662628cd98279a1972d523c3f2ebc358130ed3bdb0c6fc2d3f", "crates/perry-runtime/src/gc/policy.rs": "aea89274a017156efea3516b4f49124f48a66527a08195b662cfbf61672ac042", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" @@ -456,6 +456,12 @@ "verdict": "test_only", "why": "#[cfg(test)] `RefCell>` in `block_skip::sabotage`: the `data` base addresses of the arena BLOCKS the last full sweep skipped, recorded so tests can ask whether a given block was skipped. Block bases, not object pointers; never dereferenced, never traced, absent from shipped binaries." }, + { + "file": "crates/perry-runtime/src/gc/verify.rs", + "name": "FULL_REMEMBERED_REBUILDS_SKIPPED", + "verdict": "not_a_gc_pointer", + "why": "#10182 live-subject counter: a `Cell` count of synchronous full collections whose old-to-young remembered-set rebuild was replaced by an exact clear (`OldToYoungRememberedRebuildState::provably_empty`). A tally, never an address." + }, { "file": "crates/perry-runtime/src/gc/verify_diag.rs", "name": "LAST_VERIFY_BUDGETED_COMPLETIONS", From 5e5f09ce295bb049eb9ceb27106dbac27b013a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 21:23:05 +0200 Subject: [PATCH 09/11] docs: per-live-object cost of a synchronous full (#10182) --- docs/src/internals/garbage-collector.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/src/internals/garbage-collector.md b/docs/src/internals/garbage-collector.md index a2aea3c04f..334a09ce6a 100644 --- a/docs/src/internals/garbage-collector.md +++ b/docs/src/internals/garbage-collector.md @@ -53,6 +53,26 @@ bytes on each sweep's `[gc] blocks:` line. +**Per-live-object cost of a synchronous full.** Three parts of a full scale +with the live set, and each has a cheaper exact form: + +- *Membership.* The census answers "is this address an arena object start?" + from one object-start bitmap per censused block (one bit per 8-byte + alignment unit, allocated in 8 KiB chunks; oversized blocks keep a sorted + start list). A traced pointer field costs a search over the block fences and + one bit test. +- *Remembered-set rebuild.* When no young object is marked or pinned after the + mark and the malloc registry is empty, the old→young rebuild could only + produce an empty set, so the full installs an empty set without walking the + old generation. `PERRY_GC_DIAG=1` prints `[gc-remembered-rebuild] full + skipped=young_generation_unmarked`. +- *Sweep page accounting.* Consecutive single-page old objects are summed and + applied to their page's metadata once, before any page-index flush that + could zero it. + + + + `PERRY_GC_SCAVENGE` is on by default and lets nursery pressure route to the direct minor. `PERRY_GC_SCAVENGE_NURSERY_MB` tunes its base high-water cap, 16 MiB by default From b54a9278d95ccf1c7f34bd4a180cb095023e56ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 21:53:31 +0200 Subject: [PATCH 10/11] changelog: fragment for #10220 --- changelog.d/10220-gc-full-throughput.md | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 changelog.d/10220-gc-full-throughput.md diff --git a/changelog.d/10220-gc-full-throughput.md b/changelog.d/10220-gc-full-throughput.md new file mode 100644 index 0000000000..7b7fa2fb0b --- /dev/null +++ b/changelog.d/10220-gc-full-throughput.md @@ -0,0 +1,36 @@ +A synchronous full mark-sweep costs less per live object, in three exact +changes: + +- **Census membership from per-block start bitmaps.** `ValidPointerSet` used + to answer "is this an arena object start?" with two binary searches (over + every 1024-start run's first key, then inside the run), about 46 % of a full + mark on a live JSON tree. The census now records one object-start bitmap per + block it walks (one bit per 8-byte alignment unit, allocated in 8 KiB chunks; + oversized blocks keep a sorted start list), so a traced pointer field costs a + search over the block fences and one bit test. Interior-pointer lookups scan + the bitmap backwards. The bitmaps are 1/64 of the walked bytes instead of + 8 bytes per censused object. A single contiguous bitmap vector grew into the + allocator's large-page class and cost `records_array_1m:sparse` +4.8 MiB + peak RSS for a ~100 KB index; the chunked storage reads 68 MiB, as before. +- **No remembered-set rebuild when nothing young is live.** When the census + shows no marked or pinned young object after the mark and the malloc + registry is empty, the old-to-young rebuild can only produce an empty set, + so the full installs one without walking the old generation. + `PERRY_GC_DIAG=1` prints `[gc-remembered-rebuild] full + skipped=young_generation_unmarked`. +- **Sweep page accounting once per page.** Consecutive single-page old objects + are summed and applied to their page metadata once, always before a + page-index flush that could zero that page's accounting, instead of one + overlap vector allocation and one hash lookup per object. + +On a loop that keeps one parsed `records_array_20m.json` tree in the old +generation and calls `gc()`, a full drops from 82–83 ms to 37–39 ms: mark +43 → 25 ms, rebuild 21 → 0 ms, sweep 13 → 7 ms, census 5 ms unchanged; peak +footprint 129 → 119 MiB. The 22-row JSON matrix and the gc-ratchet gated +counters are unchanged (none of the matrix rows runs a full on this base). + +Retrying the promoted-cohort pacing bound with these costs still does not +meet the #10182 bar: at `k=1, floor 16 MB` the 20 MB parse/scan/sparse rows +reach 179 MiB (below Node's 220) but cost 266–275 ms of CPU against a 208 ms +best, and `records_array_8m:scan` has 9 ms of CPU headroom for 8 iterations, +less than one full over its tree. No pacing change is included. From d6393e790373e33740bb8919ade7ed582d1b600a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 21:56:43 +0200 Subject: [PATCH 11/11] changelog: correct the #10220 fragment's matrix and headroom claims --- changelog.d/10220-gc-full-throughput.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/changelog.d/10220-gc-full-throughput.md b/changelog.d/10220-gc-full-throughput.md index 7b7fa2fb0b..27bc82afb5 100644 --- a/changelog.d/10220-gc-full-throughput.md +++ b/changelog.d/10220-gc-full-throughput.md @@ -26,11 +26,11 @@ changes: On a loop that keeps one parsed `records_array_20m.json` tree in the old generation and calls `gc()`, a full drops from 82–83 ms to 37–39 ms: mark 43 → 25 ms, rebuild 21 → 0 ms, sweep 13 → 7 ms, census 5 ms unchanged; peak -footprint 129 → 119 MiB. The 22-row JSON matrix and the gc-ratchet gated -counters are unchanged (none of the matrix rows runs a full on this base). +footprint 129 → 119 MiB. The 22-row JSON matrix stays within ±2 % CPU with no +row worse on RSS, and the gc-ratchet gated counters are identical. Retrying the promoted-cohort pacing bound with these costs still does not meet the #10182 bar: at `k=1, floor 16 MB` the 20 MB parse/scan/sparse rows reach 179 MiB (below Node's 220) but cost 266–275 ms of CPU against a 208 ms -best, and `records_array_8m:scan` has 9 ms of CPU headroom for 8 iterations, -less than one full over its tree. No pacing change is included. +best, and `records_array_8m:scan` has 13.7 ms of CPU headroom for 8 +iterations, less than one ~30 ms full over its tree. No pacing change is included.