From 9866ed292237d288637ca6922b9b4de7a27a8725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:15:54 +0200 Subject: [PATCH 01/40] gc: answer census arena membership from per-block object-start bitmaps (#10182) --- .../perry-runtime/src/gc/tests/cycle_state.rs | 51 ++- crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/start_bitmap.rs | 214 ++++++++++ crates/perry-runtime/src/gc/trace.rs | 398 ++++++++++++------ 4 files changed, 501 insertions(+), 163 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/start_bitmap.rs diff --git a/crates/perry-runtime/src/gc/tests/cycle_state.rs b/crates/perry-runtime/src/gc/tests/cycle_state.rs index 2b5d531fc6..4fa2daa7cf 100644 --- a/crates/perry-runtime/src/gc/tests/cycle_state.rs +++ b/crates/perry-runtime/src/gc/tests/cycle_state.rs @@ -271,13 +271,13 @@ fn build_valid_pointer_set_sliced_build_preserves_contains_and_enclosing_object( } } -/// #7646: arena membership now answers from the address-ordered census runs -/// rather than a shadow `BTreeSet`, which makes RUN BOUNDARIES load-bearing. -/// Runs seal every `VALID_POINTER_ARENA_RUN_CAPACITY` (1024) starts, so the -/// final run is partial and is only sealed by `finalize()`. The sliced-build -/// test above allocates 1100 strings — enough to cross the boundary — but -/// checks only the first 16, which all live in the FIRST run: it passes -/// unchanged if every later run is lost. +/// #7646: arena membership answers from the census's own address-ordered +/// structure rather than a shadow `BTreeSet`, which makes the structure's +/// boundaries load-bearing. It was a list of 1024-start runs whose last, partial +/// run only `finalize()` sealed; since #10182 it is one start bitmap per censused +/// block, where a lost block entry or a stale fence would drop every start of +/// that block. The sliced-build test above checks only the first 16 starts, so +/// it passes unchanged if every later start is lost. /// /// This checks every start, both directions. #[test] @@ -285,8 +285,6 @@ fn valid_pointer_membership_spans_every_census_run_including_the_partial_one_764 let _guard = CopyingNurseryTestGuard::new(0); let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); - // > 2 full runs, so the last one is partial and cannot be sealed by the - // capacity check alone. let arena_strings = (0..2600).map(|_| young_leaf()).collect::>(); let (arena_object, fields) = unsafe { alloc_nursery_test_object(4) }; let arena_object = arena_object as usize; @@ -295,30 +293,29 @@ fn valid_pointer_membership_spans_every_census_run_including_the_partial_one_764 let valid_ptrs = ValidPointerSetBuilder::new().finish(); assert!( - valid_ptrs.arena_runs.len() >= 3, - "premise: the census must span several runs, got {}", - valid_ptrs.arena_runs.len() - ); - assert!( - valid_ptrs.current_arena_run.is_empty(), - "finalize() must seal the open run before the set escapes the builder; \ - {} starts would otherwise be invisible to membership", - valid_ptrs.current_arena_run.len() + !valid_ptrs.arena_blocks.is_empty(), + "premise: the census must have opened at least one block" ); assert_eq!( - valid_ptrs.arena_run_firsts.len(), - valid_ptrs.arena_runs.len(), - "the fence mirror must stay index-aligned with the runs" + valid_ptrs.arena_block_bases.len(), + valid_ptrs.arena_blocks.len(), + "the fence mirror must stay index-aligned with the census blocks" ); - for (index, run) in valid_ptrs.arena_runs.iter().enumerate() { + for (index, block) in valid_ptrs.arena_blocks.iter().enumerate() { assert_eq!( - valid_ptrs.arena_run_firsts[index], run[0], - "fence {index} must equal its run's first key" + valid_ptrs.arena_block_bases[index], block.base, + "fence {index} must equal its block's base" ); + if index > 0 { + let previous = valid_ptrs.arena_blocks[index - 1]; + assert!( + previous.base + previous.extent <= block.base, + "census blocks must be ascending and disjoint" + ); + } } - // Positive: EVERY censused start, not a prefix — a start in the last, - // partial run is the one a lost `finalize()` drops. + // Positive: EVERY censused start, not a prefix. for (index, &ptr) in arena_strings.iter().enumerate() { assert!( valid_ptrs.contains(&ptr), @@ -372,7 +369,7 @@ fn build_valid_pointer_set_finalize_is_separate_bounded_phase() { } let before_finalize = builder.snapshot_for_tests(); assert_eq!(before_finalize.phase, ValidPointerSetBuildPhase::Finalize); - assert!(before_finalize.current_arena_run_len > 0 || before_finalize.arena_run_count > 0); + assert!(before_finalize.arena_block_count > 0); assert!(!builder.step(0)); assert_eq!( diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index ec960abbbc..f09c1ec962 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -64,6 +64,7 @@ mod shadow_stack_ops; mod shape_descriptor_authority; mod shape_keys_descriptor_edge; mod smoke; +mod start_bitmap; mod step_bounds; pub(super) mod support; mod survival_diag; diff --git a/crates/perry-runtime/src/gc/tests/start_bitmap.rs b/crates/perry-runtime/src/gc/tests/start_bitmap.rs new file mode 100644 index 0000000000..05c5647a41 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/start_bitmap.rs @@ -0,0 +1,214 @@ +//! #10182: the census answers arena membership from one object-start bitmap +//! per censused block (sorted start lists for oversized blocks). +//! +//! Every case plants a population on a fresh thread (so the arenas start +//! empty), builds the production census, and compares `contains` and +//! `enclosing_object` against an oracle derived from an independent arena walk +//! for every address the census covers. The sabotaged twin shifts every bitmap +//! probe by one alignment unit and shows the comparison notices. + +use super::super::*; +use super::support::*; +use crate::gc::trace::start_bitmap_sabotage; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("start-bitmap test thread must not panic"); +} + +/// Plant a population that exercises the bitmap's edges: runs of minimal +/// objects (consecutive bits across word boundaries), objects of assorted and +/// odd sizes, a 600 KB object inside a 1 MB block (an interior pointer deep in +/// it floors across many zero words), an oversized block (sorted start list), +/// nursery leaves and nursery objects. +unsafe fn plant_population() -> Vec { + let mut planted = Vec::new(); + for i in 0..9000usize { + let payload = match i % 7 { + 0 | 1 | 2 => 0, + 3 => 8, + 4 => 13, + 5 => 40, + _ => 200 + (i % 5) * 64, + }; + planted.push(crate::arena::arena_alloc_gc_old(payload, 8, GC_TYPE_STRING) as usize); + } + planted.push(crate::arena::arena_alloc_gc_old(600 * 1024, 8, GC_TYPE_STRING) as usize); + for _ in 0..200 { + planted.push(crate::arena::arena_alloc_gc_old(24, 8, GC_TYPE_STRING) as usize); + } + planted.push(crate::arena::arena_alloc_gc_old( + 2 * crate::arena::BLOCK_SIZE + 4096, + 8, + GC_TYPE_STRING, + ) as usize); + for _ in 0..3000 { + planted.push(young_leaf()); + planted.push(alloc_nursery_test_object(3).0 as usize); + } + planted +} + +/// `(user pointer, total size)` of every walkable arena object, ascending. +fn oracle_objects() -> Vec<(usize, usize)> { + let mut cursor = crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::Address); + let mut objects = Vec::new(); + while let Some((header, _)) = cursor.next() { + let size = unsafe { (*(header as *const GcHeader)).size as usize }; + objects.push((header as usize + GC_HEADER_SIZE, size)); + } + objects +} + +#[derive(Default, Debug)] +struct Comparison { + queries: usize, + contains_mismatches: usize, + enclosing_mismatches: usize, + enclosing_hits: usize, + /// An enclosing hit whose start lies at least one bitmap word (64 units) + /// below the query: the floor search had to cross zero words. + enclosing_hits_across_words: usize, + sorted_blocks: usize, + bitmap_blocks: usize, +} + +/// Query every address (step 4, so both aligned and unaligned ones) from each +/// census block's base to 64 bytes past its walked extent. +fn compare_with_oracle(valid: &ValidPointerSet, objects: &[(usize, usize)]) -> Comparison { + let starts: std::collections::HashSet = objects.iter().map(|&(u, _)| u).collect(); + let mut cmp = Comparison::default(); + for block in &valid.arena_blocks { + if block.sorted { + cmp.sorted_blocks += 1; + } else { + cmp.bitmap_blocks += 1; + } + let mut addr = block.base; + while addr < block.base + block.extent + 64 { + cmp.queries += 1; + if valid.contains(&addr) != starts.contains(&addr) { + cmp.contains_mismatches += 1; + } + let floor = objects.partition_point(|&(u, _)| u <= addr); + let expected = floor + .checked_sub(1) + .map(|i| objects[i]) + .filter(|&(u, size)| addr >= u && addr < u + size.saturating_sub(GC_HEADER_SIZE)) + .map(|(u, _)| u); + let got = valid.enclosing_object(addr); + if got != expected { + cmp.enclosing_mismatches += 1; + } + if let Some(start) = got { + cmp.enclosing_hits += 1; + if addr - start >= 64 * 8 + GC_HEADER_SIZE { + cmp.enclosing_hits_across_words += 1; + } + } + addr += 4; + } + } + cmp +} + +#[test] +fn start_bitmap_membership_and_floors_match_an_independent_arena_walk() { + run_isolated(|| { + let _planted = unsafe { plant_population() }; + let objects = oracle_objects(); + let valid = ValidPointerSetBuilder::new().finish(); + + assert_eq!( + valid.arena_count, + objects.len(), + "the census must record exactly the walkable objects" + ); + let cmp = compare_with_oracle(&valid, &objects); + assert!( + cmp.sorted_blocks >= 1, + "premise: an oversized block: {cmp:?}" + ); + assert!( + cmp.bitmap_blocks >= 2, + "premise: several bitmap blocks: {cmp:?}" + ); + assert!( + cmp.enclosing_hits_across_words > 0, + "premise: some floor search must cross bitmap words: {cmp:?}" + ); + assert_eq!(cmp.contains_mismatches, 0, "{cmp:?}"); + assert_eq!(cmp.enclosing_mismatches, 0, "{cmp:?}"); + // Index size is the point of the change: well under the 8 B/object the + // start runs cost. + assert!( + valid.arena_index_bytes() < objects.len() * 8, + "bitmap index {} B for {} objects", + valid.arena_index_bytes(), + objects.len() + ); + }); +} + +#[test] +fn sabotaged_bitmap_probe_is_caught_by_the_oracle() { + run_isolated(|| { + let _planted = unsafe { plant_population() }; + let objects = oracle_objects(); + let valid = ValidPointerSetBuilder::new().finish(); + let cmp = { + let _sabotage = start_bitmap_sabotage::Guard::arm(); + compare_with_oracle(&valid, &objects) + }; + assert!( + cmp.contains_mismatches > 0, + "a bitmap probing the neighbouring unit must disagree with the walk: {cmp:?}" + ); + }); +} + +/// A real full collection over the planted population keeps a rooted object in +/// a bitmap block and the rooted oversized object in the sorted block, and +/// reclaims a dead neighbour of the small one per object. +#[test] +fn a_full_collection_marks_through_the_bitmap_and_the_sorted_list() { + run_isolated(|| { + let planted = unsafe { plant_population() }; + let big = *planted + .iter() + .find(|&&u| { + let size = unsafe { (*header_from_user_ptr(u as *const u8)).size as usize }; + size > crate::arena::BLOCK_SIZE + }) + .expect("premise: an oversized object"); + let small = planted[4321]; + let mut small_root = string_bits(small); + let mut big_root = string_bits(big); + js_gc_register_global_root(&mut small_root as *mut u64 as i64); + js_gc_register_global_root(&mut big_root as *mut u64 as i64); + let _ = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); + for user in [small, big] { + let header = header_from_user_ptr(user as *const u8); + assert_eq!( + unsafe { (*header).obj_type }, + GC_TYPE_STRING, + "rooted object {user:#x} must survive the full" + ); + assert!(crate::arena::pointer_in_old_gen(user)); + } + assert_eq!( + unsafe { (*header_from_user_ptr(planted[4322] as *const u8)).obj_type }, + 0, + "the unrooted neighbour of the rooted small object is swept" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 0901a6549c..a24d0a07a4 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -94,42 +94,68 @@ crate::perry_thread_local! { const { std::cell::UnsafeCell::new(Vec::new()) }; } -const VALID_POINTER_ARENA_RUN_CAPACITY: usize = 1024; +/// Census blocks whose walked extent exceeds this keep a sorted start list +/// instead of a start bitmap: an oversized block (one large allocation rounded +/// up to a `BLOCK_SIZE` multiple) holds a handful of objects, and a bitmap over +/// its whole extent would be mostly zero words. +const CENSUS_BITMAP_MAX_EXTENT: usize = crate::arena::BLOCK_SIZE; + +/// Arena object starts sit on 8-byte boundaries relative to their block's +/// `data` pointer: `ArenaObjectCursor::next_budgeted` rounds every header +/// offset up to a multiple of 8 before reading it, and the census consumes +/// exactly the headers that cursor yields. +const CENSUS_START_ALIGN_SHIFT: u32 = 3; + +/// One censused arena block, in address order (#10182). +#[derive(Clone, Copy, Debug)] +pub(super) struct CensusStartBlock { + /// The block's `data` address as the census cursor snapshotted it. + pub(super) base: usize, + /// Bytes the census walked (`offset` in the snapshot). No censused header + /// starts at or past `base + extent`. + pub(super) extent: usize, + /// Global arena block index (`u32::MAX` when unknown). + pub(super) block_idx: u32, + /// Bitmap blocks: index of the block's first word in `start_bits`. + /// Sorted blocks: index of the block's first start in `large_starts`. + pub(super) first: usize, + /// Bitmap blocks: word count. Sorted blocks: start count. + pub(super) len: usize, + pub(super) sorted: bool, +} pub(crate) struct ValidPointerSet { - /// Arena-only start pointers in address-ordered runs — **the exact arena - /// membership set**, not merely an index for `enclosing_object`'s floor - /// lookups. `ArenaObjectCursorBuilder::new(ArenaWalkOrder::Address)` hands - /// the census headers over in ascending address order, so each run is - /// sorted by construction and a floor lookup that lands on the query IS - /// the membership answer. + /// **The exact arena membership set**, one entry per censused arena block + /// in ascending address order. `ArenaObjectCursorBuilder::new( + /// ArenaWalkOrder::Address)` hands the census the blocks in address order + /// and each block's headers in address order, so the table is sorted by + /// construction and a block never appears twice. + /// + /// Membership used to answer from address-ordered runs of user pointers + /// (1024 per run, sealed at block boundaries): a binary search over every + /// run's first key, then a second binary search inside the run — about 21 + /// cache-missing probes per traced pointer field, which was ~46 % of a full + /// mark on a live 20 MB JSON tree. The runs replaced a shadow `BTreeSet` + /// (#7592) and cost 8 bytes per censused object. /// - /// This used to be shadowed by a parallel `BTreeSet` over the same - /// addresses. That set cost one B-tree insert per live arena object with - /// nothing to show for it: the runs already held the same data in the same - /// order. On `json_pipeline` 500k the shadow cost **245.5 ms of a 748.3 ms - /// full collection** (`phase_us.build_valid_pointer_set`), 12.6% of the - /// `build_out` phase, and ~40 MB of transient peak heap (#7592). - pub(super) arena_runs: Vec>, - /// `arena_runs[i]`'s global arena block index (`u32::MAX` for runs pushed - /// without one). The census seals a run at every block boundary, so a - /// run never straddles two blocks and a membership hit names its block - /// for free (#10182). - pub(super) arena_run_blocks: Vec, - pub(super) current_arena_run_block: u32, + /// Now each block carries an **object-start bitmap** (1 bit per 8-byte + /// alignment unit of its walked extent, 16 KB for a full 1 MB block) and a + /// query is a search over the block fences (`arena_block_bases`, one entry + /// per block) plus one bit test. Oversized blocks keep a sorted start list + /// (`large_starts`), since they hold a few objects over many megabytes. + pub(super) arena_blocks: Vec, + /// `arena_blocks[i].base`, mirrored into one contiguous vector so the + /// block-level binary search reads 8-byte fences only. + pub(super) arena_block_bases: Vec, + /// Concatenated start bitmaps of the bitmap blocks. Bit `k` of a block's + /// bitmap is set iff a censused header starts at `base + (k << 3)`. + pub(super) start_bits: Vec, + /// Concatenated ascending start lists (user pointers) of the sorted blocks. + pub(super) large_starts: Vec, /// Per-block census facts and trace reachability (#10182). Disarmed /// unless this set was built by the production census walk. pub(super) block_census: BlockCensus, - /// `arena_runs[i].first()`, mirrored into one contiguous vector so the - /// run-level binary search reads 8-byte fences instead of chasing a - /// `Vec` header per probe. At 500k `json_pipeline` records this is ~4k - /// entries (32 KB, L2-resident) against 33 MB of run storage, and it is - /// what keeps the membership lookup competitive with the B-tree probe it - /// replaces (#7592). - pub(super) arena_run_firsts: Vec, - pub(super) current_arena_run: Vec, - /// Live count of pushed arena starts (sealed runs + the open one), kept so - /// `lookup_count` stays O(1). + /// Live count of pushed arena starts, kept so `lookup_count` stays O(1). pub(super) arena_count: usize, /// Exact membership for **malloc-tracked** objects only, which have no /// address order to exploit. A B-tree avoids hash-table rebuilds in tiny @@ -170,12 +196,11 @@ pub(crate) struct ValidPointerSet { impl ValidPointerSet { pub(super) fn new() -> Self { Self { - arena_runs: Vec::new(), - arena_run_blocks: Vec::new(), - current_arena_run_block: u32::MAX, + arena_blocks: Vec::new(), + arena_block_bases: Vec::new(), + start_bits: Vec::new(), + large_starts: Vec::new(), block_census: BlockCensus::disarmed(), - arena_run_firsts: Vec::new(), - current_arena_run: Vec::with_capacity(VALID_POINTER_ARENA_RUN_CAPACITY), arena_count: 0, malloc_lookup: std::collections::BTreeSet::new(), range_min: usize::MAX, @@ -186,34 +211,77 @@ impl ValidPointerSet { } } - /// Caller must guarantee that pushes happen in ascending address - /// order — `ValidPointerSetBuilder` does so via `ArenaObjectCursor` - /// in address order. `block_idx` is the start's global arena block; the - /// open run is sealed whenever it changes, so no run straddles two blocks - /// (#10182). - pub(super) fn push_arena_in_block(&mut self, ptr: usize, block_idx: u32) { + /// Open census block `block_idx`: `data`/`offset` exactly as the census + /// cursor snapshotted it. Every start pushed until the next call belongs to + /// this block. Blocks must be opened in ascending address order. + pub(super) fn begin_arena_block(&mut self, block_idx: u32, data: usize, offset: usize) { if self.classifier_mode { return; // #6179: no exact census in classifier mode } - if block_idx != self.current_arena_run_block { - self.seal_current_arena_run(); - self.current_arena_run_block = block_idx; - } - if let Some(previous) = self - .current_arena_run - .last() - .copied() - .or_else(|| self.arena_runs.last().and_then(|run| run.last()).copied()) - { - debug_assert!(previous <= ptr); + if let Some(previous) = self.arena_blocks.last() { + assert!( + previous.base.saturating_add(previous.extent) <= data, + "census blocks must arrive in ascending, non-overlapping address order: \ + {:#x}+{:#x} then {data:#x}", + previous.base, + previous.extent + ); } + let sorted = offset > CENSUS_BITMAP_MAX_EXTENT; + let (first, len) = if sorted { + (self.large_starts.len(), 0) + } else { + let bits = offset.div_ceil(1 << CENSUS_START_ALIGN_SHIFT); + let words = bits.div_ceil(64); + let first = self.start_bits.len(); + self.start_bits.resize(first + words, 0); + (first, words) + }; + self.arena_block_bases.push(data); + self.arena_blocks.push(CensusStartBlock { + base: data, + extent: offset, + block_idx, + first, + len, + sorted, + }); + } - self.record_pointer_range(ptr); - self.current_arena_run.push(ptr); - self.arena_count += 1; - if self.current_arena_run.len() >= VALID_POINTER_ARENA_RUN_CAPACITY { - self.seal_current_arena_run(); + /// Record a censused arena start (user pointer) in the block opened by the + /// last `begin_arena_block`. Starts arrive in ascending address order — + /// `ValidPointerSetBuilder` feeds them from `ArenaObjectCursor` in address + /// order. + pub(super) fn push_arena(&mut self, ptr: usize) { + if self.classifier_mode { + return; // #6179: no exact census in classifier mode } + let block = self + .arena_blocks + .last_mut() + .expect("an arena start is pushed only inside an opened census block"); + let header_offset = ptr.wrapping_sub(block.base).wrapping_sub(GC_HEADER_SIZE); + // A start the bitmap cannot represent would be a silent false negative + // (swept live), so the cursor's alignment contract is checked, not + // assumed. One predictable compare per censused object. + assert!( + header_offset < block.extent + && header_offset & ((1 << CENSUS_START_ALIGN_SHIFT) - 1) == 0, + "census start {ptr:#x} is outside or misaligned in its block \ + {:#x}+{:#x}", + block.base, + block.extent + ); + if block.sorted { + debug_assert!(self.large_starts.last().is_none_or(|&last| last < ptr)); + self.large_starts.push(ptr); + block.len += 1; + } else { + let bit = header_offset >> CENSUS_START_ALIGN_SHIFT; + self.start_bits[block.first + (bit >> 6)] |= 1u64 << (bit & 63); + } + self.arena_count += 1; + self.record_pointer_range(ptr); } pub(super) fn push_malloc(&mut self, ptr: usize) { @@ -236,8 +304,13 @@ impl ValidPointerSet { pub(super) fn tenured_nursery_bytes(&self) -> usize { self.tenured_nursery_bytes } - pub(super) fn finalize(&mut self) { - self.seal_current_arena_run(); + /// Transient heap bytes the arena membership index holds (fences, block + /// table, bitmaps, oversized-block start lists). + pub(super) fn arena_index_bytes(&self) -> usize { + self.arena_blocks.capacity() * std::mem::size_of::() + + self.arena_block_bases.capacity() * std::mem::size_of::() + + self.start_bits.capacity() * std::mem::size_of::() + + self.large_starts.capacity() * std::mem::size_of::() } #[inline(always)] @@ -250,18 +323,6 @@ impl ValidPointerSet { } } - fn seal_current_arena_run(&mut self) { - if self.current_arena_run.is_empty() { - return; - } - let sealed = std::mem::take(&mut self.current_arena_run); - // Non-empty by the guard above, so the fence mirror stays index-aligned - // with `arena_runs` — `arena_run_firsts[i] == arena_runs[i][0]`. - self.arena_run_firsts.push(sealed[0]); - self.arena_runs.push(sealed); - self.arena_run_blocks.push(self.current_arena_run_block); - } - /// Cheap O(1) range-rejection prefilter. Most stack words and /// register spills are not heap pointers; if the candidate falls /// outside `[range_min, range_max]` it cannot match either region @@ -292,11 +353,10 @@ impl ValidPointerSet { // censused address range. return false; } - // Exact lookup. Arena starts answer from the address-ordered census - // runs (a floor lookup that lands ON the query is membership); only - // malloc-tracked starts, which have no usable order, need the B-tree. - // Arena first because arena hits dominate every workload that reaches - // here — a malloc pointer pays one extra run-level binary search. + // Exact lookup. Arena starts answer from the per-block start bitmaps; + // only malloc-tracked starts, which have no usable order, need the + // B-tree. Arena first because arena hits dominate every workload that + // reaches here — a malloc pointer pays one extra fence search. let exact = self.arena_start_censused(*ptr) || (!self.malloc_lookup.is_empty() && self.malloc_lookup.contains(ptr)); // #6179 differential verification (PERRY_GC_VERIFY_CLASSIFIER=1): @@ -336,13 +396,14 @@ impl ValidPointerSet { /// iteration. Find the largest entry `<= query`, then validate via /// the GcHeader's size field. pub(crate) fn enclosing_object(&self, ptr: usize) -> Option { - let (candidate, run) = self.find_arena_floor_run(ptr)?; + let block = self.census_block_at(ptr)?; + let candidate = self.floor_start_in_block(block, ptr)?; unsafe { let header = (candidate as *const u8).sub(GC_HEADER_SIZE) as *const GcHeader; let total = (*header).size as usize; let payload_end = candidate + total.saturating_sub(GC_HEADER_SIZE); if ptr >= candidate && ptr < payload_end { - self.note_run_reached(run); + self.block_census.note_reached(block.block_idx); Some(candidate) } else { None @@ -350,57 +411,85 @@ impl ValidPointerSet { } } - /// A census hit in run `run`: its block was reached by the trace (#10182). - /// See `block_skip`'s module doc for why every mark passes through here. + /// The census block whose base is the greatest one `<= ptr`, if any. + /// `ptr` may still lie past that block's walked extent. #[inline(always)] - fn note_run_reached(&self, run: usize) { - if let Some(&block_idx) = self.arena_run_blocks.get(run) { - self.block_census.note_reached(block_idx); + fn census_block_at(&self, ptr: usize) -> Option<&CensusStartBlock> { + let idx = self.arena_block_bases.partition_point(|&base| base <= ptr); + if idx == 0 { + return None; } + self.arena_blocks.get(idx - 1) } - /// Exact arena membership: the census runs are address-ordered, so `ptr` - /// was censused iff its floor is itself. - /// - /// **Load-bearing ordering requirement, which the `BTreeSet` this replaced - /// did not have.** The B-tree was complete after every `push_arena`, so a - /// mid-build query merely saw fewer entries. The runs are only complete - /// once `finalize()` has sealed `current_arena_run` — up to - /// `VALID_POINTER_ARENA_RUN_CAPACITY` censused starts are invisible before - /// that. A membership query on an unsealed set is therefore a FALSE - /// NEGATIVE, and a false negative here is not a missed optimisation: the - /// conservative scan drops the root, the object is swept live, and the - /// failure surfaces cycles later as `TypeError: value is not a function`. - /// - /// The builder's phase machine guarantees this today (`Finalize` precedes - /// `Done`, and the set escapes only through `finish()`), so the assert is - /// free in release. It exists so that a phase added after `Finalize`, or a - /// caller that queries a partially-built set, fails a test instead of - /// corrupting the heap. Verified to fire: skipping the seal in `finalize` - /// trips it with "4 censused starts are invisible to this lookup". + /// Exact arena membership: a census hit also records that the trace + /// reached the hit's block (#10182; see `block_skip`'s module doc for why + /// every mark passes through here). #[inline] fn arena_start_censused(&self, ptr: usize) -> bool { - debug_assert!( - self.current_arena_run.is_empty(), - "arena membership queried before finalize() sealed the open run: \ - {} censused starts are invisible to this lookup", - self.current_arena_run.len() - ); - match self.find_arena_floor_run(ptr) { - Some((floor, run)) if floor == ptr => { - self.note_run_reached(run); - true + let Some(block) = self.census_block_at(ptr) else { + return false; + }; + let hit = if block.sorted { + self.large_starts[block.first..block.first + block.len] + .binary_search(&ptr) + .is_ok() + } else { + // Below `base + GC_HEADER_SIZE` the subtraction wraps past every + // extent, so one compare rejects both ends of the block. + let header_offset = ptr.wrapping_sub(block.base).wrapping_sub(GC_HEADER_SIZE); + if header_offset >= block.extent + || header_offset & ((1 << CENSUS_START_ALIGN_SHIFT) - 1) != 0 + { + false + } else { + let bit = header_offset >> CENSUS_START_ALIGN_SHIFT; + #[cfg(test)] + let bit = start_bitmap_sabotage::shift_probe(bit); + self.start_bits + .get(block.first + (bit >> 6)) + .is_some_and(|word| (word >> (bit & 63)) & 1 != 0) } - _ => false, + }; + if hit { + self.block_census.note_reached(block.block_idx); } + hit } - fn find_arena_floor_run(&self, ptr: usize) -> Option<(usize, usize)> { - let idx = self.arena_run_firsts.partition_point(|&first| first <= ptr); - if idx == 0 { + /// Greatest censused start (user pointer) in `block` that is `<= ptr`. + fn floor_start_in_block(&self, block: &CensusStartBlock, ptr: usize) -> Option { + if block.sorted { + let starts = &self.large_starts[block.first..block.first + block.len]; + return Self::find_floor(starts, ptr); + } + if block.extent == 0 { return None; } - Self::find_floor(&self.arena_runs[idx - 1], ptr).map(|floor| (floor, idx - 1)) + // A start `s <= ptr` has its header at `s - GC_HEADER_SIZE`, so the + // highest candidate header offset is `ptr - base - GC_HEADER_SIZE`, + // clamped to the last walked byte. + let header_offset = ptr.checked_sub(block.base)?.checked_sub(GC_HEADER_SIZE)?; + let bit = header_offset.min(block.extent - 1) >> CENSUS_START_ALIGN_SHIFT; + let mut word_idx = bit >> 6; + let top = bit & 63; + let keep = if top == 63 { + u64::MAX + } else { + (1u64 << (top + 1)) - 1 + }; + let mut word = self.start_bits[block.first + word_idx] & keep; + loop { + if word != 0 { + let floor_bit = word_idx * 64 + (63 - word.leading_zeros() as usize); + return Some(block.base + (floor_bit << CENSUS_START_ALIGN_SHIFT) + GC_HEADER_SIZE); + } + if word_idx == 0 { + return None; + } + word_idx -= 1; + word = self.start_bits[block.first + word_idx]; + } } pub(super) fn find_floor(sorted: &[usize], ptr: usize) -> Option { @@ -415,6 +504,42 @@ impl ValidPointerSet { } } +/// Sabotage switch for the start-bitmap tests: shifts every bitmap probe by +/// one alignment unit, so a test can show its membership oracle notices a +/// bitmap that answers for the wrong address. Test builds only. +#[cfg(test)] +pub(crate) mod start_bitmap_sabotage { + use std::cell::Cell; + + thread_local! { + static SHIFT_PROBE: Cell = const { Cell::new(false) }; + } + + #[inline] + pub(crate) fn shift_probe(bit: usize) -> usize { + if SHIFT_PROBE.with(Cell::get) { + bit + 1 + } else { + bit + } + } + + /// Arms the shifted probe until the guard drops. + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(SHIFT_PROBE.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + SHIFT_PROBE.with(|s| s.set(self.0)); + } + } +} + /// Build a set of all valid user-space pointers (pointers returned to callers). /// Used to validate candidates found during conservative stack scanning. pub(crate) fn build_valid_pointer_set() -> ValidPointerSet { @@ -449,8 +574,7 @@ pub(super) enum ValidPointerSetBuildPhase { pub(super) struct ValidPointerSetBuilderSnapshot { pub(super) phase: ValidPointerSetBuildPhase, pub(super) arena_setup_blocks: usize, - pub(super) arena_run_count: usize, - pub(super) current_arena_run_len: usize, + pub(super) arena_block_count: usize, pub(super) lookup_count: usize, pub(super) malloc_index: usize, } @@ -535,7 +659,6 @@ impl ValidPointerSetBuilder { return false; } self.set.block_census.flush_block(); - self.set.finalize(); self.phase = ValidPointerSetBuildPhase::Done; return true; } @@ -573,34 +696,38 @@ impl ValidPointerSetBuilder { } return false; }; - if self.census_armed { - if block_idx != self.census_block_idx { - self.census_block_idx = block_idx; - if let Some((_, data, offset)) = self - .arena_cursor - .as_ref() - .and_then(crate::arena::ArenaObjectCursor::current_block_extent) - { + if block_idx != self.census_block_idx { + self.census_block_idx = block_idx; + if let Some((_, data, offset)) = self + .arena_cursor + .as_ref() + .and_then(crate::arena::ArenaObjectCursor::current_block_extent) + { + self.set.begin_arena_block( + u32::try_from(block_idx).unwrap_or(u32::MAX), + data, + offset, + ); + if self.census_armed { self.set.block_census.begin_block(block_idx, data, offset); } } + } + if self.census_armed { unsafe { self.set .block_census .note_header(header_ptr as *const GcHeader); } } - self.record_arena_header(header_ptr, block_idx); + self.record_arena_header(header_ptr); } false } - fn record_arena_header(&mut self, header_ptr: *mut u8, block_idx: usize) { + fn record_arena_header(&mut self, header_ptr: *mut u8) { let user_ptr = unsafe { header_ptr.add(GC_HEADER_SIZE) }; - self.set.push_arena_in_block( - user_ptr as usize, - u32::try_from(block_idx).unwrap_or(u32::MAX), - ); + self.set.push_arena(user_ptr as usize); unsafe { let header = header_ptr as *const GcHeader; let flags = (*header).gc_flags; @@ -639,8 +766,7 @@ impl ValidPointerSetBuilder { .arena_cursor_builder .as_ref() .map_or(0, crate::arena::ArenaObjectCursorBuilder::inspected_blocks), - arena_run_count: self.set.arena_runs.len(), - current_arena_run_len: self.set.current_arena_run.len(), + arena_block_count: self.set.arena_blocks.len(), lookup_count: self.set.lookup_count(), malloc_index: self.malloc_index, } From 13dc830f25ea83b04225031f40b8d394c865b251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:21:32 +0200 Subject: [PATCH 02/40] gc: test-only arena index size accessor --- crates/perry-runtime/src/gc/trace.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index a24d0a07a4..f546f70c43 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -306,6 +306,7 @@ impl ValidPointerSet { } /// Transient heap bytes the arena membership index holds (fences, block /// table, bitmaps, oversized-block start lists). + #[cfg(test)] pub(super) fn arena_index_bytes(&self) -> usize { self.arena_blocks.capacity() * std::mem::size_of::() + self.arena_block_bases.capacity() * std::mem::size_of::() From 6f3bc9e5fc44cd8f67787ecf84b9c879171619f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:30:41 +0200 Subject: [PATCH 03/40] gc: start-bitmap test fixup --- crates/perry-runtime/src/gc/tests/start_bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/gc/tests/start_bitmap.rs b/crates/perry-runtime/src/gc/tests/start_bitmap.rs index 05c5647a41..4538848b33 100644 --- a/crates/perry-runtime/src/gc/tests/start_bitmap.rs +++ b/crates/perry-runtime/src/gc/tests/start_bitmap.rs @@ -197,7 +197,7 @@ fn a_full_collection_marks_through_the_bitmap_and_the_sorted_list() { GcTriggerKind::OldGenBytes, )); for user in [small, big] { - let header = header_from_user_ptr(user as *const u8); + let header = unsafe { header_from_user_ptr(user as *const u8) }; assert_eq!( unsafe { (*header).obj_type }, GC_TYPE_STRING, From d75c51c47e35d149b128f69720dbda5c745c3de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:44:07 +0200 Subject: [PATCH 04/40] gc: allocate census start bitmaps in 8 KiB chunks --- crates/perry-runtime/src/gc/trace.rs | 84 +++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index f546f70c43..f8e75ee301 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -100,6 +100,17 @@ crate::perry_thread_local! { /// its whole extent would be mostly zero words. const CENSUS_BITMAP_MAX_EXTENT: usize = crate::arena::BLOCK_SIZE; +/// Start bitmaps are allocated in chunks of this many words (8 KiB), one or two +/// per block. A bitmap kept in one contiguous vector grew past the allocator's +/// small and medium size classes and made it commit a fresh large page: +/// `records_array_1m:sparse` read +4.8 MiB peak RSS for a ~100 KB index. The +/// start runs this replaces were 8 KiB vectors too. +const CENSUS_BITMAP_CHUNK_WORDS: usize = 1024; +const CENSUS_BITMAP_CHUNK_WORD_SHIFT: u32 = 10; +/// Chunks a bitmap block can need: `CENSUS_BITMAP_MAX_EXTENT` bytes at one bit +/// per 8 bytes is `2 * CENSUS_BITMAP_CHUNK_WORDS` words. +const CENSUS_BITMAP_MAX_CHUNKS: usize = 2; + /// Arena object starts sit on 8-byte boundaries relative to their block's /// `data` pointer: `ArenaObjectCursor::next_budgeted` rounds every header /// offset up to a multiple of 8 before reading it, and the census consumes @@ -116,7 +127,11 @@ pub(super) struct CensusStartBlock { pub(super) extent: usize, /// Global arena block index (`u32::MAX` when unknown). pub(super) block_idx: u32, - /// Bitmap blocks: index of the block's first word in `start_bits`. + /// Bitmap blocks: the block's bitmap chunks (owned by + /// `ValidPointerSet::start_bitmap_chunks`); word `w` is word + /// `w & (CENSUS_BITMAP_CHUNK_WORDS - 1)` of chunk `w >> 10`. Null past the + /// block's word count. + pub(super) chunks: [*mut u64; CENSUS_BITMAP_MAX_CHUNKS], /// Sorted blocks: index of the block's first start in `large_starts`. pub(super) first: usize, /// Bitmap blocks: word count. Sorted blocks: start count. @@ -124,6 +139,20 @@ pub(super) struct CensusStartBlock { pub(super) sorted: bool, } +impl CensusStartBlock { + /// Word `word_idx` of this bitmap block's start bitmap. + /// + /// # Safety + /// `self` is a bitmap block of a live `ValidPointerSet` and + /// `word_idx < self.len`. + #[inline(always)] + unsafe fn word_ptr(&self, word_idx: usize) -> *mut u64 { + debug_assert!(!self.sorted && word_idx < self.len); + self.chunks[word_idx >> CENSUS_BITMAP_CHUNK_WORD_SHIFT] + .add(word_idx & (CENSUS_BITMAP_CHUNK_WORDS - 1)) + } +} + pub(crate) struct ValidPointerSet { /// **The exact arena membership set**, one entry per censused arena block /// in ascending address order. `ArenaObjectCursorBuilder::new( @@ -147,9 +176,11 @@ pub(crate) struct ValidPointerSet { /// `arena_blocks[i].base`, mirrored into one contiguous vector so the /// block-level binary search reads 8-byte fences only. pub(super) arena_block_bases: Vec, - /// Concatenated start bitmaps of the bitmap blocks. Bit `k` of a block's - /// bitmap is set iff a censused header starts at `base + (k << 3)`. - pub(super) start_bits: Vec, + /// Storage of the bitmap blocks' start bitmaps, in 8 KiB chunks (see + /// `CENSUS_BITMAP_CHUNK_WORDS`). Bit `k` of a block's bitmap is set iff a + /// censused header starts at `base + (k << 3)`. A chunk's heap buffer never + /// moves, so `CensusStartBlock::chunks` may point into it. + pub(super) start_bitmap_chunks: Vec>, /// Concatenated ascending start lists (user pointers) of the sorted blocks. pub(super) large_starts: Vec, /// Per-block census facts and trace reachability (#10182). Disarmed @@ -198,7 +229,7 @@ impl ValidPointerSet { Self { arena_blocks: Vec::new(), arena_block_bases: Vec::new(), - start_bits: Vec::new(), + start_bitmap_chunks: Vec::new(), large_starts: Vec::new(), block_census: BlockCensus::disarmed(), arena_count: 0, @@ -228,20 +259,29 @@ impl ValidPointerSet { ); } let sorted = offset > CENSUS_BITMAP_MAX_EXTENT; + let mut chunks = [std::ptr::null_mut(); CENSUS_BITMAP_MAX_CHUNKS]; let (first, len) = if sorted { (self.large_starts.len(), 0) } else { let bits = offset.div_ceil(1 << CENSUS_START_ALIGN_SHIFT); let words = bits.div_ceil(64); - let first = self.start_bits.len(); - self.start_bits.resize(first + words, 0); - (first, words) + for (index, chunk) in chunks.iter_mut().enumerate() { + let start = index * CENSUS_BITMAP_CHUNK_WORDS; + if start >= words { + break; + } + let mut storage = vec![0u64; (words - start).min(CENSUS_BITMAP_CHUNK_WORDS)]; + *chunk = storage.as_mut_ptr(); + self.start_bitmap_chunks.push(storage); + } + (0, words) }; self.arena_block_bases.push(data); self.arena_blocks.push(CensusStartBlock { base: data, extent: offset, block_idx, + chunks, first, len, sorted, @@ -278,7 +318,11 @@ impl ValidPointerSet { block.len += 1; } else { let bit = header_offset >> CENSUS_START_ALIGN_SHIFT; - self.start_bits[block.first + (bit >> 6)] |= 1u64 << (bit & 63); + // SAFETY: `header_offset < extent` (asserted above), so the word + // index is below the block's word count. + unsafe { + *block.word_ptr(bit >> 6) |= 1u64 << (bit & 63); + } } self.arena_count += 1; self.record_pointer_range(ptr); @@ -310,7 +354,12 @@ impl ValidPointerSet { pub(super) fn arena_index_bytes(&self) -> usize { self.arena_blocks.capacity() * std::mem::size_of::() + self.arena_block_bases.capacity() * std::mem::size_of::() - + self.start_bits.capacity() * std::mem::size_of::() + + self + .start_bitmap_chunks + .iter() + .map(|chunk| chunk.capacity() * std::mem::size_of::()) + .sum::() + + self.start_bitmap_chunks.capacity() * std::mem::size_of::>() + self.large_starts.capacity() * std::mem::size_of::() } @@ -447,9 +496,12 @@ impl ValidPointerSet { let bit = header_offset >> CENSUS_START_ALIGN_SHIFT; #[cfg(test)] let bit = start_bitmap_sabotage::shift_probe(bit); - self.start_bits - .get(block.first + (bit >> 6)) - .is_some_and(|word| (word >> (bit & 63)) & 1 != 0) + // SAFETY: `header_offset < extent`, so the word index is below + // the block's word count (the sabotaged probe in test builds + // may step one bit past it, still inside the last word). + let word_idx = bit >> 6; + word_idx < block.len + && unsafe { (*block.word_ptr(word_idx) >> (bit & 63)) & 1 != 0 } } }; if hit { @@ -479,7 +531,9 @@ impl ValidPointerSet { } else { (1u64 << (top + 1)) - 1 }; - let mut word = self.start_bits[block.first + word_idx] & keep; + // SAFETY: `bit` derives from an offset below `extent`, and every + // later index is smaller. + let mut word = unsafe { *block.word_ptr(word_idx) } & keep; loop { if word != 0 { let floor_bit = word_idx * 64 + (63 - word.leading_zeros() as usize); @@ -489,7 +543,7 @@ impl ValidPointerSet { return None; } word_idx -= 1; - word = self.start_bits[block.first + word_idx]; + word = unsafe { *block.word_ptr(word_idx) }; } } From 7e3b008a26b99a4920850b2a4a6512b068b7b928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 19:59:53 +0200 Subject: [PATCH 05/40] gc: skip a full's remembered-set rebuild when nothing young is marked (#10182) --- crates/perry-runtime/src/arena/mod.rs | 4 +- crates/perry-runtime/src/arena/walk.rs | 10 + crates/perry-runtime/src/gc/cycle.rs | 9 + .../src/gc/tests/full_rebuild_skip.rs | 197 ++++++++++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + crates/perry-runtime/src/gc/tests/oldgen.rs | 12 +- .../perry-runtime/src/gc/trace/block_skip.rs | 42 ++++ crates/perry-runtime/src/gc/verify.rs | 63 ++++++ 8 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 785f7dc4d1..3df3c590d8 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -103,8 +103,8 @@ pub use walk::{ pub(crate) use walk::{ arena_block_snapshots, arena_telemetry_snapshot, general_block_in_recent_window, general_block_sizes, old_arena_walk_all_headers_filtered, young_allocation_census, - ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder, ArenaTelemetrySnapshot, - ArenaWalkOrder, + young_block_count, ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder, + ArenaTelemetrySnapshot, ArenaWalkOrder, }; // reset.rs diff --git a/crates/perry-runtime/src/arena/walk.rs b/crates/perry-runtime/src/arena/walk.rs index 7f44b82f06..2031b6c9c1 100644 --- a/crates/perry-runtime/src/arena/walk.rs +++ b/crates/perry-runtime/src/arena/walk.rs @@ -905,6 +905,16 @@ pub fn general_block_count() -> usize { ARENA.with(|arena| unsafe { (*arena.get()).blocks.len() }) } +/// Global block indices `0..young_block_count()` are the young generation: +/// the general (Eden) arena's blocks, then both survivor semispaces' — the +/// order `arena_block_snapshots` and the object cursors use. +pub(crate) fn young_block_count() -> usize { + let g = ARENA.with(|arena| unsafe { (*arena.get()).blocks.len() }); + let s0 = SURVIVOR_ARENA_0.with(|arena| unsafe { (*arena.get()).blocks.len() }); + let s1 = SURVIVOR_ARENA_1.with(|arena| unsafe { (*arena.get()).blocks.len() }); + g + s0 + s1 +} + /// Per-block `size` for the general (nursery-Eden) arena, indexed by /// general block index (`0..general_block_count()`, tombstones report /// size 0). Used by the evacuation policy to translate "this block's diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 826ea115d9..2a85720828 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1294,6 +1294,15 @@ impl GcCycleState { .as_mut() .expect("atomic finalize state exists"); let rebuild = state.remembered_rebuild.get_or_insert_with(|| { + // #10182: nothing young is marked and no malloc object + // exists, so the walk could only insert nothing. + if !budgeted + && valid_ptrs.is_some_and(|ptrs| { + full_remembered_rebuild_provably_empty(&ptrs.block_census) + }) + { + return OldToYoungRememberedRebuildState::provably_empty(); + } let skip = if budgeted { None } else { diff --git a/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs b/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs new file mode 100644 index 0000000000..8ba5a7423f --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs @@ -0,0 +1,197 @@ +//! #10182: a synchronous full replaces its old→young remembered-set rebuild by +//! an exact clear when the rebuild could only produce an empty set — no marked +//! or pinned young object, and no malloc object. +//! +//! Each case runs on a fresh thread (empty arenas, empty malloc registry). The +//! protective cases plant an old→young or old→malloc edge that ONLY the rebuild +//! can recover — a raw store with no write barrier, so the pre-cycle dirty +//! snapshot does not cover it — and check the remembered set covers it after +//! the full. Their sabotaged twins force the skip and show the edge is lost. + +use super::super::*; +use super::support::*; +use crate::gc::trace::block_skip::sabotage; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + let _barriers = GeneratedWriteBarrierTestGuard::active(); + reset_global_roots(); + reset_remembered_set(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("full-rebuild-skip test thread must not panic"); +} + +fn synchronous_full() -> GcCycleTrace { + gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot { + kind: GcTriggerKind::OldGenBytes, + steps_before: Some(GcStepSnapshot::current()), + }) + .trace + .expect("full GC trace requested") +} + +/// A rooted old parent with one field. The root cell is leaked so its address +/// stays valid for the thread's lifetime. +fn rooted_old_parent() -> (usize, *mut u64) { + let (parent, fields) = unsafe { alloc_old_test_object(1) }; + let root: &'static mut u64 = Box::leak(Box::new(ptr_bits(parent as usize))); + js_gc_register_global_root(root as *mut u64 as i64); + (parent as usize, fields) +} + +/// Store `child_bits` into the parent's field WITHOUT a write barrier. +fn raw_store(parent: usize, fields: *mut u64, child_bits: u64) { + unsafe { + *fields = child_bits; + layout_note_slot(parent, 0, child_bits); + } +} + +#[test] +fn a_full_with_no_live_young_object_skips_the_rebuild() { + run_isolated(|| { + let (parent, fields) = rooted_old_parent(); + let (old_child, _) = unsafe { alloc_old_test_object(0) }; + raw_store(parent, fields, ptr_bits(old_child as usize)); + // Young garbage: the young generation is in use but nothing in it lives. + for _ in 0..512 { + let _ = unsafe { alloc_nursery_test_object(2) }; + } + let skips = crate::gc::full_remembered_rebuilds_skipped(); + + let trace = synchronous_full(); + + assert_eq!( + crate::gc::full_remembered_rebuilds_skipped(), + skips + 1, + "a full with an unmarked young generation must skip the rebuild" + ); + assert_eq!(trace.old_to_young_rebuild_objects_scanned, 0); + assert_eq!( + remembered_set_size(), + 0, + "the skipped rebuild leaves an empty set" + ); + assert_eq!(verify_old_to_young_edges_collect().missing_edges, 0); + + // The mutator's next old→young store is still remembered and survives + // a copying minor. + let child = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize; + assert!(crate::arena::pointer_in_nursery(child)); + raw_store(parent, fields, ptr_bits(child)); + js_write_barrier_slot(ptr_bits(parent), fields as u64, ptr_bits(child)); + let _ = collect_minor_trace(GcTriggerKind::Direct); + let slot_child = (unsafe { *fields } & POINTER_MASK) as usize; + assert_eq!( + unsafe { (*header_from_user_ptr(slot_child as *const u8)).obj_type }, + GC_TYPE_OBJECT, + "the barrier-recorded young child must survive the minor" + ); + }); +} + +/// The young child is reachable only through an old parent, stored without a +/// barrier: the rebuild is the only thing that can remember it. +fn plant_unbarriered_young_edge() -> (usize, *mut u64, usize) { + let (parent, fields) = rooted_old_parent(); + let child = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize; + assert!(crate::arena::pointer_in_nursery(child)); + raw_store(parent, fields, ptr_bits(child)); + assert_eq!( + remembered_set_size(), + 0, + "premise: no barrier recorded the edge" + ); + (parent, fields, child) +} + +#[test] +fn a_live_young_child_keeps_the_rebuild_and_its_edge() { + run_isolated(|| { + let (_parent, fields, child) = plant_unbarriered_young_edge(); + let skips = crate::gc::full_remembered_rebuilds_skipped(); + + let trace = synchronous_full(); + + assert_eq!(crate::gc::full_remembered_rebuilds_skipped(), skips); + assert!(trace.old_to_young_rebuild_objects_scanned > 0); + let verify = verify_old_to_young_edges_collect(); + assert!(verify.checked_old_to_young_edges > 0, "premise: {verify:?}"); + assert_eq!( + verify.missing_edges, 0, + "the rebuild must remember the edge" + ); + assert_eq!(unsafe { *fields } & POINTER_MASK, child as u64); + }); +} + +#[test] +fn sabotaged_skip_loses_an_unbarriered_young_edge() { + run_isolated(|| { + let _ = plant_unbarriered_young_edge(); + { + let _sabotage = sabotage::Guard::arm(sabotage::FORCE_REBUILD_SKIP); + let _ = synchronous_full(); + } + let verify = verify_old_to_young_edges_collect(); + assert!( + verify.missing_edges > 0, + "a wrongly skipped rebuild must leave the young edge unremembered: {verify:?}" + ); + }); +} + +/// A malloc-registry child of an old parent, stored without a barrier, with an +/// empty young generation: only the malloc guard stops the skip. +fn plant_unbarriered_malloc_edge() -> *mut u64 { + activate_malloc_registry_for_tests(); + let (parent, fields) = rooted_old_parent(); + let symbol = alloc_tracked_test_symbol() as usize; + assert!(malloc_user_ptr_tracked(symbol as *mut u8)); + raw_store(parent, fields, ptr_bits(symbol)); + assert_eq!( + remembered_set_size(), + 0, + "premise: no barrier recorded the edge" + ); + fields +} + +#[test] +fn a_live_malloc_child_keeps_the_rebuild_and_its_edge() { + run_isolated(|| { + let _fields = plant_unbarriered_malloc_edge(); + let skips = crate::gc::full_remembered_rebuilds_skipped(); + + let _ = synchronous_full(); + + assert_eq!(crate::gc::full_remembered_rebuilds_skipped(), skips); + let verify = verify_old_to_young_edges_collect(); + assert!(verify.checked_old_to_young_edges > 0, "premise: {verify:?}"); + assert_eq!( + verify.missing_edges, 0, + "the rebuild must remember the malloc edge" + ); + }); +} + +#[test] +fn sabotaged_skip_loses_an_unbarriered_malloc_edge() { + run_isolated(|| { + let _ = plant_unbarriered_malloc_edge(); + { + let _sabotage = sabotage::Guard::arm(sabotage::FORCE_REBUILD_SKIP); + let _ = synchronous_full(); + } + let verify = verify_old_to_young_edges_collect(); + assert!( + verify.missing_edges > 0, + "a wrongly skipped rebuild must leave the malloc edge unremembered: {verify:?}" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index f09c1ec962..b94cb3b4a3 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -27,6 +27,7 @@ mod forwarding_target_validation; mod forwarding_verification; mod fromspace_protect; mod fromspace_scan; +mod full_rebuild_skip; mod global_bootstrap; mod global_sink_isolation; mod handle_bound_method_name; diff --git a/crates/perry-runtime/src/gc/tests/oldgen.rs b/crates/perry-runtime/src/gc/tests/oldgen.rs index 2b26f3fc7b..c6d3bce7e4 100644 --- a/crates/perry-runtime/src/gc/tests/oldgen.rs +++ b/crates/perry-runtime/src/gc/tests/oldgen.rs @@ -1305,7 +1305,14 @@ fn test_minor_skips_whole_heap_old_to_young_rebuild() { // A full cycle DOES run the rebuild — the counter proves it is wired and // scales with the (now-large) heap, so the minor's 0 is a genuine skip - // rather than the counter being dead. + // rather than the counter being dead. #10182: a full whose young generation + // holds nothing live replaces the rebuild by an exact clear, so keep one + // pinned young object alive across it. + let (young, _young_fields) = unsafe { alloc_nursery_test_object(1) }; + let young_header = unsafe { header_from_user_ptr(young as *const u8) }; + unsafe { + crate::gc::pin_object(young_header); + } let full_outcome = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot { kind: GcTriggerKind::Direct, steps_before: Some(GcStepSnapshot::current()), @@ -1316,6 +1323,9 @@ fn test_minor_skips_whole_heap_old_to_young_rebuild() { "a full cycle must walk the whole heap for the RS rebuild (got {}, expected >= {OLD_OBJECTS})", full_trace.old_to_young_rebuild_objects_scanned, ); + unsafe { + crate::gc::unpin_object(young_header); + } for header in old_headers { unsafe { diff --git a/crates/perry-runtime/src/gc/trace/block_skip.rs b/crates/perry-runtime/src/gc/trace/block_skip.rs index a04e9a3b04..22d53fb18b 100644 --- a/crates/perry-runtime/src/gc/trace/block_skip.rs +++ b/crates/perry-runtime/src/gc/trace/block_skip.rs @@ -62,6 +62,10 @@ pub(crate) struct CensusBlock { pub(crate) censused: bool, /// Some object in the block needs the per-object sweep path. pub(crate) obligation: bool, + /// Some header in the block was already MARKED or PINNED when the census + /// read it (a subset of `obligation`, kept apart for + /// `young_generation_unmarked`). + pub(crate) premarked: bool, } /// Per-block census facts and trace reachability for one cycle's @@ -135,6 +139,7 @@ impl BlockCensus { bytes: 0, censused: true, obligation: false, + premarked: false, }; } @@ -151,6 +156,7 @@ impl BlockCensus { let exceptional_flags = (flags ^ GC_FLAG_ARENA) & (GC_FLAG_ARENA | GC_FLAG_MARKED | GC_FLAG_PINNED | GC_FLAG_FORWARDED) != 0; + let premarked = flags & (GC_FLAG_MARKED | GC_FLAG_PINNED) != 0; let raw_f64_array = obj_type == GC_TYPE_ARRAY && (*header)._reserved & (GC_ARRAY_RAW_F64_LAYOUT | GC_ARRAY_RAW_F64_HOLES) != 0; let type_obligation = self.obligation_by_type[obj_type as usize]; @@ -158,6 +164,7 @@ impl BlockCensus { block.objects += 1; block.bytes += size; block.obligation |= exceptional_flags | type_obligation | raw_f64_array; + block.premarked |= premarked; } /// Fold the current block into the per-index table. Called at every block @@ -213,6 +220,38 @@ impl BlockCensus { any.then_some(skip) } + /// After the mark of a synchronous full: does the young generation (Eden + /// and both survivor spaces) hold **no** marked or pinned object? + /// + /// Every in-use young block must be censused, unchanged since the census + /// (no allocate-black birth, no block created after it), free of headers + /// that were already marked or pinned when censused, and unreached by the + /// trace. An unreached block holds no object the trace marked (see this + /// module's doc: every census-built mark passes a membership query that + /// records its block), so every young object is then garbage. `false` when + /// the census is disarmed or anything is uncertain. + pub(crate) fn young_generation_unmarked(&self) -> bool { + if !self.armed { + return false; + } + let snapshots = crate::arena::arena_block_snapshots(); + let young = crate::arena::young_block_count().min(snapshots.len()); + snapshots[..young] + .iter() + .enumerate() + .all(|(block_idx, snapshot)| { + if snapshot.data == 0 || snapshot.offset == 0 { + return true; + } + self.block(block_idx).is_some_and(|block| { + !block.premarked + && !self.reached(block_idx) + && block.data == snapshot.data + && block.end == snapshot.data.saturating_add(snapshot.offset) + }) + }) + } + pub(crate) fn block(&self, block_idx: usize) -> Option { self.blocks.get(block_idx).copied().filter(|b| b.censused) } @@ -231,6 +270,9 @@ pub(crate) mod sabotage { pub(crate) const FORGET_REACHED: u8 = 1; pub(crate) const FORGET_OBLIGATIONS: u8 = 2; + /// `verify::full_remembered_rebuild_provably_empty` answers true whatever + /// the heap holds. + pub(crate) const FORCE_REBUILD_SKIP: u8 = 4; thread_local! { static SABOTAGE: Cell = const { Cell::new(0) }; diff --git a/crates/perry-runtime/src/gc/verify.rs b/crates/perry-runtime/src/gc/verify.rs index 36e6038046..48a642a65d 100644 --- a/crates/perry-runtime/src/gc/verify.rs +++ b/crates/perry-runtime/src/gc/verify.rs @@ -550,6 +550,48 @@ unsafe fn remember_retained_old_to_young_slots( }); } +crate::perry_thread_local! { + /// Synchronous full collections whose old→young remembered-set rebuild was + /// replaced by an exact clear because the young generation held no marked + /// object (#10182). Live-subject counter for the tests and the diag line. + static FULL_REMEMBERED_REBUILDS_SKIPPED: std::cell::Cell = + const { std::cell::Cell::new(0) }; +} + +/// Running count of [`FULL_REMEMBERED_REBUILDS_SKIPPED`] on this thread. +pub(crate) fn full_remembered_rebuilds_skipped() -> u64 { + FULL_REMEMBERED_REBUILDS_SKIPPED.with(std::cell::Cell::get) +} + +/// #10182: after a synchronous full's mark, can the old→young remembered-set +/// rebuild only produce an empty set? +/// +/// The rebuild remembers a slot of a marked (or pinned) old parent exactly when +/// its child classifies as young (nursery) or is a registered malloc object +/// (`remembered_child_needs_tracking`). A marked parent's strong child is +/// marked too, and the rebuild skips weak slots exactly as the trace does. So: +/// +/// * if no young object is marked or pinned (`young_generation_unmarked`), +/// every young child a marked parent could name is garbage the sweep is +/// about to reclaim, and no mutator runs in between to make one live; and +/// * if the malloc registry is empty — the same premise the copying minor's +/// `skip_remembering` uses — there is no malloc child at all, +/// +/// then the walk can insert nothing that names a live object, and the +/// remembered set this full leaves behind is exactly empty. The pre-cycle +/// dirty snapshot repair (`restore_surviving_dirty_coverage`) still runs in +/// reclaim as before. A budgeted cycle has no census, so it never qualifies. +pub(super) fn full_remembered_rebuild_provably_empty(census: &super::trace::BlockCensus) -> bool { + #[cfg(test)] + if super::trace::block_skip::sabotage::get() + & super::trace::block_skip::sabotage::FORCE_REBUILD_SKIP + != 0 + { + return census.is_armed(); + } + census.young_generation_unmarked() && MALLOC_STATE.with(|s| s.borrow().objects.is_empty()) +} + pub(super) struct OldToYoungRememberedRebuildState { require_marked: bool, sticky: StickyRememberedSet, @@ -586,6 +628,27 @@ impl OldToYoungRememberedRebuildState { } } + /// The rebuild of a full whose result is provably empty + /// (`full_remembered_rebuild_provably_empty`): no walk, an empty set. + pub(super) fn provably_empty() -> Self { + FULL_REMEMBERED_REBUILDS_SKIPPED.with(|c| c.set(c.get().saturating_add(1))); + if crate::gc::gc_diag_enabled() { + eprintln!( + "[gc-remembered-rebuild] full skipped=young_generation_unmarked skips_total={}", + full_remembered_rebuilds_skipped() + ); + } + Self { + require_marked: true, + sticky: StickyRememberedSet::default(), + arena_cursor: None, + arena_done: true, + malloc_index: 0, + objects_scanned: 0, + done: true, + } + } + /// Number of heap objects this whole-heap rebuild walk has visited. Used /// by the GC trace to prove that minors do NOT run this O(all-objects) /// walk (#6181): full cycles report the walked object count, minors 0. From d9aae42a34a1a8b69784a2e2ec4c332fb9ce037f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 20:11:23 +0200 Subject: [PATCH 06/40] gc: full-rebuild-skip tests check the known-live parent --- .../src/gc/tests/full_rebuild_skip.rs | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs b/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs index 8ba5a7423f..8675d1600b 100644 --- a/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs +++ b/crates/perry-runtime/src/gc/tests/full_rebuild_skip.rs @@ -44,6 +44,21 @@ fn rooted_old_parent() -> (usize, *mut u64) { (parent as usize, fields) } +/// The old→young edge verifier checks a parent only while it is marked or +/// pinned (or already remembered); after a completed full no mark is left, so +/// check the known-live parent explicitly. +fn verify_live_parent(parent: usize) -> OldYoungEdgeVerifyStats { + let header = unsafe { header_from_user_ptr(parent as *const u8) }; + unsafe { + (*header).gc_flags |= GC_FLAG_MARKED; + } + let stats = verify_old_to_young_edges_collect(); + unsafe { + (*header).gc_flags &= !GC_FLAG_MARKED; + } + stats +} + /// Store `child_bits` into the parent's field WITHOUT a write barrier. fn raw_store(parent: usize, fields: *mut u64, child_bits: u64) { unsafe { @@ -77,7 +92,8 @@ fn a_full_with_no_live_young_object_skips_the_rebuild() { 0, "the skipped rebuild leaves an empty set" ); - assert_eq!(verify_old_to_young_edges_collect().missing_edges, 0); + let verify = verify_live_parent(parent); + assert_eq!(verify.missing_edges, 0, "{verify:?}"); // The mutator's next old→young store is still remembered and survives // a copying minor. @@ -113,14 +129,14 @@ fn plant_unbarriered_young_edge() -> (usize, *mut u64, usize) { #[test] fn a_live_young_child_keeps_the_rebuild_and_its_edge() { run_isolated(|| { - let (_parent, fields, child) = plant_unbarriered_young_edge(); + let (parent, fields, child) = plant_unbarriered_young_edge(); let skips = crate::gc::full_remembered_rebuilds_skipped(); let trace = synchronous_full(); assert_eq!(crate::gc::full_remembered_rebuilds_skipped(), skips); assert!(trace.old_to_young_rebuild_objects_scanned > 0); - let verify = verify_old_to_young_edges_collect(); + let verify = verify_live_parent(parent); assert!(verify.checked_old_to_young_edges > 0, "premise: {verify:?}"); assert_eq!( verify.missing_edges, 0, @@ -133,12 +149,12 @@ fn a_live_young_child_keeps_the_rebuild_and_its_edge() { #[test] fn sabotaged_skip_loses_an_unbarriered_young_edge() { run_isolated(|| { - let _ = plant_unbarriered_young_edge(); + let (parent, _, _) = plant_unbarriered_young_edge(); { let _sabotage = sabotage::Guard::arm(sabotage::FORCE_REBUILD_SKIP); let _ = synchronous_full(); } - let verify = verify_old_to_young_edges_collect(); + let verify = verify_live_parent(parent); assert!( verify.missing_edges > 0, "a wrongly skipped rebuild must leave the young edge unremembered: {verify:?}" @@ -148,7 +164,7 @@ fn sabotaged_skip_loses_an_unbarriered_young_edge() { /// A malloc-registry child of an old parent, stored without a barrier, with an /// empty young generation: only the malloc guard stops the skip. -fn plant_unbarriered_malloc_edge() -> *mut u64 { +fn plant_unbarriered_malloc_edge() -> usize { activate_malloc_registry_for_tests(); let (parent, fields) = rooted_old_parent(); let symbol = alloc_tracked_test_symbol() as usize; @@ -159,19 +175,19 @@ fn plant_unbarriered_malloc_edge() -> *mut u64 { 0, "premise: no barrier recorded the edge" ); - fields + parent } #[test] fn a_live_malloc_child_keeps_the_rebuild_and_its_edge() { run_isolated(|| { - let _fields = plant_unbarriered_malloc_edge(); + let parent = plant_unbarriered_malloc_edge(); let skips = crate::gc::full_remembered_rebuilds_skipped(); let _ = synchronous_full(); assert_eq!(crate::gc::full_remembered_rebuilds_skipped(), skips); - let verify = verify_old_to_young_edges_collect(); + let verify = verify_live_parent(parent); assert!(verify.checked_old_to_young_edges > 0, "premise: {verify:?}"); assert_eq!( verify.missing_edges, 0, @@ -183,12 +199,12 @@ fn a_live_malloc_child_keeps_the_rebuild_and_its_edge() { #[test] fn sabotaged_skip_loses_an_unbarriered_malloc_edge() { run_isolated(|| { - let _ = plant_unbarriered_malloc_edge(); + let parent = plant_unbarriered_malloc_edge(); { let _sabotage = sabotage::Guard::arm(sabotage::FORCE_REBUILD_SKIP); let _ = synchronous_full(); } - let verify = verify_old_to_young_edges_collect(); + let verify = verify_live_parent(parent); assert!( verify.missing_edges > 0, "a wrongly skipped rebuild must leave the malloc edge unremembered: {verify:?}" From 420a641a69700331a1f34f7c599098e48a34b9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 20:24:02 +0200 Subject: [PATCH 07/40] gc: apply a full sweep's old-page accounting once per page (#10182) --- crates/perry-runtime/src/arena/mod.rs | 13 +- .../perry-runtime/src/arena/page_meta/mod.rs | 4 + .../src/arena/page_meta/sweep_tally.rs | 87 ++++++ .../src/gc/oldgen/sweep_batch.rs | 7 + .../src/gc/oldgen/sweep_objects.rs | 93 +++++- crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/sweep_page_tally.rs | 265 ++++++++++++++++++ .../perry-runtime/src/gc/trace/block_skip.rs | 3 + 8 files changed, 455 insertions(+), 18 deletions(-) create mode 100644 crates/perry-runtime/src/arena/page_meta/sweep_tally.rs create mode 100644 crates/perry-runtime/src/gc/tests/sweep_page_tally.rs diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 3df3c590d8..a54decd1b0 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -151,12 +151,13 @@ pub(crate) use page_meta::{ classify_heap_space_in_range, generation_page_for_addr, materialize_all_promoted_page_runs, old_arena_block_range_index, old_arena_block_ranges, old_arena_page_index_remove_object, old_arena_source_blocks_for_pages, old_arena_walk_objects_on_pages, old_object_page_overlaps, - old_page_account_dirty_slot, old_page_account_dirty_slots, old_page_account_promoted_object, - old_page_account_swept_object, old_page_clear_dirty, old_page_mark_dirty, - old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle, - old_pages_reset_sweep_accounting, record_arena_object_start, unregister_old_object_pages, - unregister_old_objects_batch, HeapGeneration, HeapSpace, OldArenaPageObjectCursor, - OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, + old_object_single_page, old_page_account_dirty_slot, old_page_account_dirty_slots, + old_page_account_promoted_object, old_page_account_swept_object, old_page_account_swept_tally, + old_page_clear_dirty, old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, + old_pages_begin_gc_cycle, old_pages_reset_sweep_accounting, record_arena_object_start, + unregister_old_object_pages, unregister_old_objects_batch, HeapGeneration, HeapSpace, + OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, + OldPageSweepTally, }; #[cfg(test)] diff --git a/crates/perry-runtime/src/arena/page_meta/mod.rs b/crates/perry-runtime/src/arena/page_meta/mod.rs index f2f0b180b6..7575141fac 100644 --- a/crates/perry-runtime/src/arena/page_meta/mod.rs +++ b/crates/perry-runtime/src/arena/page_meta/mod.rs @@ -139,7 +139,11 @@ impl PageGenerationCache { const PAGE_GENERATION_CACHE_WAYS: usize = 4; mod page_class; +mod sweep_tally; pub(crate) use page_class::*; +pub(crate) use sweep_tally::{ + old_object_single_page, old_page_account_swept_tally, OldPageSweepTally, +}; #[cfg(test)] mod tests; diff --git a/crates/perry-runtime/src/arena/page_meta/sweep_tally.rs b/crates/perry-runtime/src/arena/page_meta/sweep_tally.rs new file mode 100644 index 0000000000..1a43e2488c --- /dev/null +++ b/crates/perry-runtime/src/arena/page_meta/sweep_tally.rs @@ -0,0 +1,87 @@ +//! Per-page batching of a sweep's old-generation page accounting (#10182). +//! +//! `old_page_account_swept_object` costs one heap-allocated overlap vector and +//! one page-meta hash lookup per swept old object. A full sweep over a live +//! 20 MB JSON tree calls it ~585k times, and it was about three quarters of the +//! sweep. The sweep walks each block in address order, so consecutive objects +//! share a page: summing their deltas and applying the sum once per page gives +//! the same page meta, because every field involved is a plain sum and +//! `refresh_policy_bits` is a pure recompute of the page's own fields. +//! +//! The one interleaving that could observe the difference is the batched +//! page-index removal (`unregister_old_objects_batch`), which zeroes a page's +//! sweep accounting when its last object leaves. The sweep therefore applies +//! its tally before every such flush, which keeps the order of resets and sums +//! on every page exactly as the one-by-one calls had it. + +use super::*; + +/// Sweep accounting deltas of consecutive old objects that each lie wholly on +/// one page. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct OldPageSweepTally { + pub(crate) live_bytes: usize, + pub(crate) live_objects: usize, + pub(crate) pinned_bytes: usize, + pub(crate) pinned_objects: usize, + pub(crate) dead_bytes: usize, + pub(crate) dead_objects: usize, +} + +impl OldPageSweepTally { + #[inline(always)] + pub(crate) fn add(&mut self, total_size: usize, live: bool, pinned: bool) { + if live { + self.live_bytes += total_size; + self.live_objects += 1; + if pinned { + self.pinned_bytes += total_size; + self.pinned_objects += 1; + } + } else { + self.dead_bytes += total_size; + self.dead_objects += 1; + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.live_objects == 0 && self.dead_objects == 0 + } +} + +/// The page an old object lies on when it lies wholly on one page. +#[inline(always)] +pub(crate) fn old_object_single_page(header_addr: usize, total_size: usize) -> Option { + if header_addr == 0 || total_size == 0 { + return None; + } + let first_page = generation_page_for_addr(header_addr); + (first_page == generation_page_for_addr(header_addr + total_size - 1)).then_some(first_page) +} + +/// Apply `tally` to `page` exactly as the same objects' one-by-one +/// `old_page_account_swept_object` calls would have. +pub(crate) fn old_page_account_swept_tally(page: usize, tally: &OldPageSweepTally) { + if tally.is_empty() { + return; + } + OLD_GEN_PAGE_META.with(|meta| { + let mut meta = meta.borrow_mut(); + let page_meta = meta + .entry(page) + .or_insert_with(|| OldPageMeta::zero_for_page(page)); + page_meta.live_bytes = page_meta.live_bytes.saturating_add(tally.live_bytes); + page_meta.live_object_count = page_meta + .live_object_count + .saturating_add(tally.live_objects); + page_meta.pinned_bytes = page_meta.pinned_bytes.saturating_add(tally.pinned_bytes); + page_meta.pinned_object_count = page_meta + .pinned_object_count + .saturating_add(tally.pinned_objects); + page_meta.dead_bytes = page_meta.dead_bytes.saturating_add(tally.dead_bytes); + page_meta.dead_object_count = page_meta + .dead_object_count + .saturating_add(tally.dead_objects); + page_meta.refresh_policy_bits(); + }); +} diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs index db13089b1c..74daed90c1 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs @@ -35,6 +35,13 @@ impl PendingOldUnregister { } } + /// Will the next `defer` flush the queue (and so zero the sweep accounting + /// of every page whose last object it removes)? + #[inline] + pub(super) fn flushes_on_next_defer(&self) -> bool { + self.dead.len() + 1 >= FLUSH_AT + } + /// Remove every queued header from the page index. pub(super) fn flush(&mut self) { if self.dead.is_empty() { diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs index b488837772..0cd37bc9ab 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs @@ -7,6 +7,10 @@ pub(super) struct ArenaSweepObjectsState { cursor: crate::arena::ArenaObjectCursor, /// Dead old headers awaiting one batched page-index removal (see `sweep_batch`). pending_old_unregister: super::sweep_batch::PendingOldUnregister, + /// Page accounting of consecutive single-page old objects, applied once per + /// page (see `arena::page_meta::sweep_tally`). `usize::MAX` when empty. + old_page_tally_page: usize, + old_page_tally: crate::arena::OldPageSweepTally, block_snapshots: Vec, block_has_live: Vec, resettable_general_n: usize, @@ -71,6 +75,8 @@ impl ArenaSweepObjectsState { Self { cursor: crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::BlockIndex), pending_old_unregister: Default::default(), + old_page_tally_page: usize::MAX, + old_page_tally: Default::default(), block_snapshots, block_has_live: vec![false; n_blocks], resettable_general_n: crate::arena::general_block_count(), @@ -218,11 +224,79 @@ impl ArenaSweepObjectsState { remaining -= 1; self.process_object(header_ptr as *mut GcHeader, block_idx); } - // Never leave a dead header in the page index across a step boundary. - self.pending_old_unregister.flush(); + // Never leave a dead header in the page index, or a page's accounting + // unapplied, across a step boundary. The tally goes first: the + // unregister flush zeroes the accounting of pages it empties. + if self.page_tally_order_kept() { + self.apply_old_page_tally(); + self.pending_old_unregister.flush(); + } else { + self.pending_old_unregister.flush(); + self.apply_old_page_tally(); + } done } + /// Account one swept old object on its page(s), batching single-page + /// objects per page. + #[inline] + fn account_old_object( + &mut self, + header: *mut GcHeader, + total_size: usize, + live: bool, + pinned: bool, + ) { + match crate::arena::old_object_single_page(header as usize, total_size) { + Some(page) => { + if page != self.old_page_tally_page { + self.apply_old_page_tally(); + self.old_page_tally_page = page; + } + self.old_page_tally.add(total_size, live, pinned); + } + None => crate::arena::old_page_account_swept_object( + header as usize, + total_size, + live, + pinned, + ), + } + } + + fn apply_old_page_tally(&mut self) { + if self.old_page_tally_page != usize::MAX { + crate::arena::old_page_account_swept_tally( + self.old_page_tally_page, + &self.old_page_tally, + ); + } + self.old_page_tally_page = usize::MAX; + self.old_page_tally = Default::default(); + } + + /// The tally is applied before every page-index flush (always, outside the + /// sabotaged test). + #[inline(always)] + fn page_tally_order_kept(&self) -> bool { + #[cfg(test)] + { + use super::super::trace::block_skip::sabotage; + sabotage::get() & sabotage::FORGET_PAGE_TALLY_ORDER == 0 + } + #[cfg(not(test))] + true + } + + /// Queue a dead old header's page-index removal, applying the page tally + /// first when the queue is about to flush. + unsafe fn defer_old_unregister(&mut self, header: *mut GcHeader, total_size: usize) { + if self.pending_old_unregister.flushes_on_next_defer() && self.page_tally_order_kept() { + self.apply_old_page_tally(); + } + self.pending_old_unregister.defer(header, total_size); + } + pub(super) fn block_has_live(&self) -> &[bool] { &self.block_has_live } @@ -315,12 +389,7 @@ impl ArenaSweepObjectsState { count_in_live_census: bool, ) { if block_idx >= self.old_block_start { - crate::arena::old_page_account_swept_object( - header as usize, - (*header).size as usize, - true, - pinned, - ); + self.account_old_object(header, (*header).size as usize, true, pinned); } if block_idx < self.block_has_live.len() { self.block_has_live[block_idx] = true; @@ -386,7 +455,7 @@ impl ArenaSweepObjectsState { let total_size = (*header).size as usize; let dead_old = block_idx >= self.old_block_start; if dead_old { - crate::arena::old_page_account_swept_object(header as usize, total_size, false, false); + self.account_old_object(header, total_size, false, false); } let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE); self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64); @@ -395,7 +464,7 @@ impl ArenaSweepObjectsState { gc_type_clear_dead_payload_side_tables((*header).obj_type, user_ptr as usize); } if self.reclaim_dead_old_blocks && dead_old { - self.pending_old_unregister.defer(header, total_size); + self.defer_old_unregister(header, total_size); } else { (*header).gc_flags = flags & !(GC_FLAG_FORWARDED | GC_FLAG_MARKED); } @@ -405,7 +474,7 @@ impl ArenaSweepObjectsState { let total_size = (*header).size as usize; let dead_old = block_idx >= self.old_block_start; if dead_old { - crate::arena::old_page_account_swept_object(header as usize, total_size, false, false); + self.account_old_object(header, total_size, false, false); } let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE); self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64); @@ -414,7 +483,7 @@ impl ArenaSweepObjectsState { } finalize_dead_arena_payload(header, user_ptr, self.overflow_active); if self.reclaim_dead_old_blocks && dead_old { - self.pending_old_unregister.defer(header, total_size); + self.defer_old_unregister(header, total_size); } } } diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index b94cb3b4a3..bb3696fe35 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -69,6 +69,7 @@ mod start_bitmap; mod step_bounds; pub(super) mod support; mod survival_diag; +mod sweep_page_tally; mod teardown; mod telemetry_verifier; mod temp_roots; diff --git a/crates/perry-runtime/src/gc/tests/sweep_page_tally.rs b/crates/perry-runtime/src/gc/tests/sweep_page_tally.rs new file mode 100644 index 0000000000..c93310bcd1 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/sweep_page_tally.rs @@ -0,0 +1,265 @@ +//! #10182: a full sweep applies old-generation page accounting once per page +//! (`arena::page_meta::sweep_tally`) instead of once per object. +//! +//! The case plants an old population whose liveness is known object by object +//! — live, pinned and dead objects of assorted sizes, objects spanning pages, +//! whole pages of dead objects inside live blocks, and more dead objects than +//! one page-index flush holds — runs one synchronous full, and compares every +//! planted page's accounting with an oracle computed from the plan: a page +//! any surviving object overlaps sums its live, pinned and dead overlaps; a +//! page no survivor overlaps is emptied by the page-index removal, which zeroes +//! its accounting. The sabotaged twin applies the tally after the page-index +//! flush instead of before and shows the comparison notices. + +use super::super::*; +use super::support::*; +use crate::gc::trace::block_skip::sabotage; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("sweep page-tally test thread must not panic"); +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Fate { + Live, + Pinned, + Dead, +} + +struct Planted { + header: usize, + size: usize, + fate: Fate, +} + +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +struct PageExpect { + live_bytes: usize, + live_objects: usize, + pinned_bytes: usize, + pinned_objects: usize, + dead_bytes: usize, + dead_objects: usize, +} + +const PAGE: usize = crate::arena::GENERATION_PAGE_SIZE; + +/// Plant the population. Live objects are held by one rooted old array; pinned +/// objects are pinned. The plan ends with a run of dead objects covering whole +/// pages, so the last page the sweep visits holds no survivor. +unsafe fn plant() -> Vec { + const LIVE_SLOTS: u32 = 2048; + let (holder, elements) = alloc_old_test_array(LIVE_SLOTS); + let root: &'static mut u64 = Box::leak(Box::new(ptr_bits(holder as usize))); + js_gc_register_global_root(root as *mut u64 as i64); + let mut planted = vec![Planted { + header: holder as usize - GC_HEADER_SIZE, + size: old_test_header_and_size(holder as usize).1, + fate: Fate::Live, + }]; + let mut live_slot = 0u32; + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let mut next = || { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (state >> 33) as usize + }; + let mut allocated = 0usize; + while allocated < 3 * crate::arena::BLOCK_SIZE { + let roll = next(); + let payload = if roll % 29 == 0 { + 5000 + roll % 7000 + } else { + 8 + (roll >> 5) % 600 + }; + let fate = match (roll >> 12) % 64 { + 0 => Fate::Pinned, + 1..=7 if live_slot < LIVE_SLOTS => Fate::Live, + _ => Fate::Dead, + }; + let user = crate::arena::arena_alloc_gc_old(payload, 8, GC_TYPE_STRING) as usize; + let size = old_test_header_and_size(user).1; + match fate { + Fate::Live => { + *elements.add(live_slot as usize) = string_bits(user); + live_slot += 1; + } + Fate::Pinned => crate::gc::pin_object(header_from_user_ptr(user as *const u8)), + Fate::Dead => {} + } + planted.push(Planted { + header: user - GC_HEADER_SIZE, + size, + fate, + }); + allocated += size; + // Every so often, a run of dead objects covering whole pages. + if roll % 97 == 0 { + let mut run = 0; + while run < 3 * PAGE { + let user = crate::arena::arena_alloc_gc_old(120, 8, GC_TYPE_STRING) as usize; + let size = old_test_header_and_size(user).1; + planted.push(Planted { + header: user - GC_HEADER_SIZE, + size, + fate: Fate::Dead, + }); + run += size; + allocated += size; + } + } + } + let mut run = 0; + while run < 3 * PAGE { + let user = crate::arena::arena_alloc_gc_old(120, 8, GC_TYPE_STRING) as usize; + let size = old_test_header_and_size(user).1; + planted.push(Planted { + header: user - GC_HEADER_SIZE, + size, + fate: Fate::Dead, + }); + run += size; + } + planted +} + +fn oracle(planted: &[Planted]) -> std::collections::BTreeMap { + let mut pages: std::collections::BTreeMap = Default::default(); + for object in planted { + let end = object.header + object.size; + let mut page_base = object.header & !(PAGE - 1); + while page_base < end { + let overlap = end.min(page_base + PAGE) - object.header.max(page_base); + let (expect, survivor) = pages.entry(page_base).or_default(); + match object.fate { + Fate::Live | Fate::Pinned => { + *survivor = true; + expect.live_bytes += overlap; + expect.live_objects += 1; + if object.fate == Fate::Pinned { + expect.pinned_bytes += overlap; + expect.pinned_objects += 1; + } + } + Fate::Dead => { + expect.dead_bytes += overlap; + expect.dead_objects += 1; + } + } + page_base += PAGE; + } + } + pages +} + +#[derive(Debug, Default)] +struct Comparison { + pages: usize, + survivor_pages_with_dead: usize, + emptied_pages: usize, + multi_page_objects: usize, + dead_objects: usize, + mismatches: usize, + first_mismatch: Option<(usize, PageExpect, PageExpect)>, +} + +fn compare(planted: &[Planted]) -> Comparison { + let meta: std::collections::HashMap = + crate::arena::old_page_meta_snapshot() + .into_iter() + .map(|m| (m.page_base, m)) + .collect(); + let mut cmp = Comparison { + multi_page_objects: planted + .iter() + .filter(|o| o.header / PAGE != (o.header + o.size - 1) / PAGE) + .count(), + dead_objects: planted.iter().filter(|o| o.fate == Fate::Dead).count(), + ..Comparison::default() + }; + for (page_base, (expect, survivor)) in oracle(planted) { + cmp.pages += 1; + let expect = if survivor { + if expect.dead_objects > 0 { + cmp.survivor_pages_with_dead += 1; + } + expect + } else { + cmp.emptied_pages += 1; + PageExpect::default() + }; + let got = meta + .get(&page_base) + .map_or(PageExpect::default(), |m| PageExpect { + live_bytes: m.live_bytes, + live_objects: m.live_object_count, + pinned_bytes: m.pinned_bytes, + pinned_objects: m.pinned_object_count, + dead_bytes: m.dead_bytes, + dead_objects: m.dead_object_count, + }); + let eligible_ok = meta.get(&page_base).is_none_or(|m| { + m.evacuation_eligible + == (m.allocated_bytes > 0 + && m.live_bytes > 0 + && m.dead_bytes > 0 + && m.pinned_bytes == 0) + }); + if got != expect || !eligible_ok { + cmp.mismatches += 1; + cmp.first_mismatch.get_or_insert((page_base, expect, got)); + } + } + cmp +} + +fn full() { + let _ = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); +} + +#[test] +fn full_sweep_page_accounting_matches_the_planted_liveness() { + run_isolated(|| { + let planted = unsafe { plant() }; + full(); + let cmp = compare(&planted); + assert!( + cmp.emptied_pages >= 3, + "premise: pages with no survivor: {cmp:?}" + ); + assert!(cmp.survivor_pages_with_dead >= 100, "premise: {cmp:?}"); + assert!(cmp.multi_page_objects >= 10, "premise: {cmp:?}"); + assert!( + cmp.dead_objects > 4096, + "premise: more than one page-index flush: {cmp:?}" + ); + assert_eq!(cmp.mismatches, 0, "{cmp:?}"); + }); +} + +#[test] +fn sabotaged_tally_order_is_caught_by_the_oracle() { + run_isolated(|| { + let planted = unsafe { plant() }; + { + let _sabotage = sabotage::Guard::arm(sabotage::FORGET_PAGE_TALLY_ORDER); + full(); + } + let cmp = compare(&planted); + assert!( + cmp.mismatches > 0, + "a tally applied after the page-index flush must leave an emptied page's accounting: {cmp:?}" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/trace/block_skip.rs b/crates/perry-runtime/src/gc/trace/block_skip.rs index 22d53fb18b..dee36a19bd 100644 --- a/crates/perry-runtime/src/gc/trace/block_skip.rs +++ b/crates/perry-runtime/src/gc/trace/block_skip.rs @@ -273,6 +273,9 @@ pub(crate) mod sabotage { /// `verify::full_remembered_rebuild_provably_empty` answers true whatever /// the heap holds. pub(crate) const FORCE_REBUILD_SKIP: u8 = 4; + /// The sweep never applies its per-page accounting tally before a page-index + /// flush or a step end (it is applied only at page changes). + pub(crate) const FORGET_PAGE_TALLY_ORDER: u8 = 8; thread_local! { static SABOTAGE: Cell = const { Cell::new(0) }; From d5072f017b7d7fe51954935d1a8c8b426a38831c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 21:22:49 +0200 Subject: [PATCH 08/40] gc: classify the full-rebuild skip counter and re-pin the census window --- scripts/gc_runtime_root_holders.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 6ce5171275..7a38ba7164 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -313,7 +313,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -329,7 +329,7 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "5c151725460ffb92a55a6bee781123ef5159263b4ce5958d16570f78216e0d67", - "crates/perry-runtime/src/gc/cycle.rs": "fdef083301463ad8cec8142ca86cc4354e289c67e97bbc6caea2e8d16d4bf089", + "crates/perry-runtime/src/gc/cycle.rs": "b65e82014de18a1746fe9563f0c5bcef202f1a66e7497d51b4556b04c913db6e", "crates/perry-runtime/src/gc/mod.rs": "9dbdde7594af06d08f45f82e426b220e586429da985e38f1e11e7f01e98a2527", "crates/perry-runtime/src/gc/policy.rs": "abd08472fe71002a55a32f7a17021ab2c029d34ff2b26eb78340d1983ae939c4", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" @@ -462,6 +462,12 @@ "verdict": "test_only", "why": "#[cfg(test)] `RefCell>` in `block_skip::sabotage`: the `data` base addresses of the arena BLOCKS the last full sweep skipped, recorded so tests can ask whether a given block was skipped. Block bases, not object pointers; never dereferenced, never traced, absent from shipped binaries." }, + { + "file": "crates/perry-runtime/src/gc/verify.rs", + "name": "FULL_REMEMBERED_REBUILDS_SKIPPED", + "verdict": "not_a_gc_pointer", + "why": "#10182 live-subject counter: a `Cell` count of synchronous full collections whose old-to-young remembered-set rebuild was replaced by an exact clear (`OldToYoungRememberedRebuildState::provably_empty`). A tally, never an address." + }, { "file": "crates/perry-runtime/src/gc/verify_diag.rs", "name": "LAST_VERIFY_BUDGETED_COMPLETIONS", From c1e185ee2c5feace228e56acd9825e572e016929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 21:23:05 +0200 Subject: [PATCH 09/40] docs: per-live-object cost of a synchronous full (#10182) --- docs/src/internals/garbage-collector.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/src/internals/garbage-collector.md b/docs/src/internals/garbage-collector.md index a2aea3c04f..334a09ce6a 100644 --- a/docs/src/internals/garbage-collector.md +++ b/docs/src/internals/garbage-collector.md @@ -53,6 +53,26 @@ bytes on each sweep's `[gc] blocks:` line. +**Per-live-object cost of a synchronous full.** Three parts of a full scale +with the live set, and each has a cheaper exact form: + +- *Membership.* The census answers "is this address an arena object start?" + from one object-start bitmap per censused block (one bit per 8-byte + alignment unit, allocated in 8 KiB chunks; oversized blocks keep a sorted + start list). A traced pointer field costs a search over the block fences and + one bit test. +- *Remembered-set rebuild.* When no young object is marked or pinned after the + mark and the malloc registry is empty, the old→young rebuild could only + produce an empty set, so the full installs an empty set without walking the + old generation. `PERRY_GC_DIAG=1` prints `[gc-remembered-rebuild] full + skipped=young_generation_unmarked`. +- *Sweep page accounting.* Consecutive single-page old objects are summed and + applied to their page's metadata once, before any page-index flush that + could zero it. + + + + `PERRY_GC_SCAVENGE` is on by default and lets nursery pressure route to the direct minor. `PERRY_GC_SCAVENGE_NURSERY_MB` tunes its base high-water cap, 16 MiB by default From 01d6c85693f7f6daaae64c1222fa72dea9de728b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 21:53:31 +0200 Subject: [PATCH 10/40] changelog: fragment for #10220 --- changelog.d/10220-gc-full-throughput.md | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 changelog.d/10220-gc-full-throughput.md diff --git a/changelog.d/10220-gc-full-throughput.md b/changelog.d/10220-gc-full-throughput.md new file mode 100644 index 0000000000..7b7fa2fb0b --- /dev/null +++ b/changelog.d/10220-gc-full-throughput.md @@ -0,0 +1,36 @@ +A synchronous full mark-sweep costs less per live object, in three exact +changes: + +- **Census membership from per-block start bitmaps.** `ValidPointerSet` used + to answer "is this an arena object start?" with two binary searches (over + every 1024-start run's first key, then inside the run), about 46 % of a full + mark on a live JSON tree. The census now records one object-start bitmap per + block it walks (one bit per 8-byte alignment unit, allocated in 8 KiB chunks; + oversized blocks keep a sorted start list), so a traced pointer field costs a + search over the block fences and one bit test. Interior-pointer lookups scan + the bitmap backwards. The bitmaps are 1/64 of the walked bytes instead of + 8 bytes per censused object. A single contiguous bitmap vector grew into the + allocator's large-page class and cost `records_array_1m:sparse` +4.8 MiB + peak RSS for a ~100 KB index; the chunked storage reads 68 MiB, as before. +- **No remembered-set rebuild when nothing young is live.** When the census + shows no marked or pinned young object after the mark and the malloc + registry is empty, the old-to-young rebuild can only produce an empty set, + so the full installs one without walking the old generation. + `PERRY_GC_DIAG=1` prints `[gc-remembered-rebuild] full + skipped=young_generation_unmarked`. +- **Sweep page accounting once per page.** Consecutive single-page old objects + are summed and applied to their page metadata once, always before a + page-index flush that could zero that page's accounting, instead of one + overlap vector allocation and one hash lookup per object. + +On a loop that keeps one parsed `records_array_20m.json` tree in the old +generation and calls `gc()`, a full drops from 82–83 ms to 37–39 ms: mark +43 → 25 ms, rebuild 21 → 0 ms, sweep 13 → 7 ms, census 5 ms unchanged; peak +footprint 129 → 119 MiB. The 22-row JSON matrix and the gc-ratchet gated +counters are unchanged (none of the matrix rows runs a full on this base). + +Retrying the promoted-cohort pacing bound with these costs still does not +meet the #10182 bar: at `k=1, floor 16 MB` the 20 MB parse/scan/sparse rows +reach 179 MiB (below Node's 220) but cost 266–275 ms of CPU against a 208 ms +best, and `records_array_8m:scan` has 9 ms of CPU headroom for 8 iterations, +less than one full over its tree. No pacing change is included. From 54421a8ad650fc0c7060be79b3cc413b9cb66a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 21:56:43 +0200 Subject: [PATCH 11/40] changelog: correct the #10220 fragment's matrix and headroom claims --- changelog.d/10220-gc-full-throughput.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/changelog.d/10220-gc-full-throughput.md b/changelog.d/10220-gc-full-throughput.md index 7b7fa2fb0b..27bc82afb5 100644 --- a/changelog.d/10220-gc-full-throughput.md +++ b/changelog.d/10220-gc-full-throughput.md @@ -26,11 +26,11 @@ changes: On a loop that keeps one parsed `records_array_20m.json` tree in the old generation and calls `gc()`, a full drops from 82–83 ms to 37–39 ms: mark 43 → 25 ms, rebuild 21 → 0 ms, sweep 13 → 7 ms, census 5 ms unchanged; peak -footprint 129 → 119 MiB. The 22-row JSON matrix and the gc-ratchet gated -counters are unchanged (none of the matrix rows runs a full on this base). +footprint 129 → 119 MiB. The 22-row JSON matrix stays within ±2 % CPU with no +row worse on RSS, and the gc-ratchet gated counters are identical. Retrying the promoted-cohort pacing bound with these costs still does not meet the #10182 bar: at `k=1, floor 16 MB` the 20 MB parse/scan/sparse rows reach 179 MiB (below Node's 220) but cost 266–275 ms of CPU against a 208 ms -best, and `records_array_8m:scan` has 9 ms of CPU headroom for 8 iterations, -less than one full over its tree. No pacing change is included. +best, and `records_array_8m:scan` has 13.7 ms of CPU headroom for 8 +iterations, less than one ~30 ms full over its tree. No pacing change is included. From 2e8b1f07bc2503eb4f6497673b1b265d2ac8a158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 23:42:26 +0200 Subject: [PATCH 12/40] gc: expand a described promoted page only where a full's sweep reshapes it (#10182) A synchronous full used to expand every pending promoted page run in its constructor, including the runs of blocks its block-granular sweep then released whole: 7-8 ms of a 62 ms pacing full on records_array_20m:parse and ~18 MiB of transient page lists. The sweep now expands a page right before it invalidates the first dead header on it (PendingOldUnregister::defer), the same order invalidate_dead_old_arena_header already keeps; a page on which every object survives keeps its run, and a block reclaimed whole discards its runs unexpanded. --- crates/perry-runtime/src/arena/mod.rs | 24 +- .../perry-runtime/src/arena/page_meta/mod.rs | 45 +++- crates/perry-runtime/src/arena/tests.rs | 4 +- .../src/arena/tests_promoted_runs.rs | 9 +- crates/perry-runtime/src/gc/cycle.rs | 8 +- .../src/gc/oldgen/sweep_batch.rs | 39 +++- crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/promote_in_place.rs | 5 +- .../src/gc/tests/sweep_described_runs.rs | 221 ++++++++++++++++++ .../perry-runtime/src/gc/trace/block_skip.rs | 3 + 10 files changed, 326 insertions(+), 33 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/sweep_described_runs.rs diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index a54decd1b0..4f1a88653b 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -148,23 +148,23 @@ pub(crate) use stats::{old_gen_in_use_bytes_recomputed, old_gen_in_use_bytes_res // page_meta.rs (public + pub(crate) classification/page-meta API) pub(crate) use page_meta::{ arena_header_is_object_start, classify_heap_generation, classify_heap_space, - classify_heap_space_in_range, generation_page_for_addr, materialize_all_promoted_page_runs, - old_arena_block_range_index, old_arena_block_ranges, old_arena_page_index_remove_object, - old_arena_source_blocks_for_pages, old_arena_walk_objects_on_pages, old_object_page_overlaps, - old_object_single_page, old_page_account_dirty_slot, old_page_account_dirty_slots, - old_page_account_promoted_object, old_page_account_swept_object, old_page_account_swept_tally, - old_page_clear_dirty, old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, - old_pages_begin_gc_cycle, old_pages_reset_sweep_accounting, record_arena_object_start, - unregister_old_object_pages, unregister_old_objects_batch, HeapGeneration, HeapSpace, - OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, - OldPageSweepTally, + classify_heap_space_in_range, generation_page_for_addr, + materialize_promoted_page_runs_for_object, old_arena_block_range_index, old_arena_block_ranges, + old_arena_page_index_remove_object, old_arena_source_blocks_for_pages, + old_arena_walk_objects_on_pages, old_object_page_overlaps, old_object_single_page, + old_page_account_dirty_slot, old_page_account_dirty_slots, old_page_account_promoted_object, + old_page_account_swept_object, old_page_account_swept_tally, old_page_clear_dirty, + old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle, + old_pages_reset_sweep_accounting, record_arena_object_start, unregister_old_object_pages, + unregister_old_objects_batch, HeapGeneration, HeapSpace, OldArenaPageObjectCursor, + OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, OldPageSweepTally, }; #[cfg(test)] pub(crate) use page_meta::{ deferred_old_page_registrations_len, generation_page_base, old_arena_page_index_clear_for_tests, old_page_meta_for_tests, - old_page_meta_snapshot_calls_for_tests, pending_promoted_page_runs, register_block_space, - register_promoted_page_run, reset_old_page_meta_snapshot_calls_for_tests, + old_page_meta_snapshot_calls_for_tests, pending_promoted_page_runs, promoted_page_run_pending, + register_block_space, register_promoted_page_run, reset_old_page_meta_snapshot_calls_for_tests, DEFERRED_OLD_PAGE_REGISTRATION_CAP, GENERATION_CLASS_SHIFT, GENERATION_PAGE_SIZE, }; diff --git a/crates/perry-runtime/src/arena/page_meta/mod.rs b/crates/perry-runtime/src/arena/page_meta/mod.rs index 7575141fac..21732bb279 100644 --- a/crates/perry-runtime/src/arena/page_meta/mod.rs +++ b/crates/perry-runtime/src/arena/page_meta/mod.rs @@ -759,16 +759,37 @@ pub(crate) fn materialize_promoted_page_runs(pages: impl IntoIterator usize { OLD_GEN_PAGE_PROMOTED_RUNS.with(|runs| runs.borrow().len()) } +/// Is `page`'s object list still DESCRIBED by a pending run? Tests only. +#[cfg(test)] +pub(crate) fn promoted_page_run_pending(page: usize) -> bool { + OLD_GEN_PAGE_PROMOTED_RUNS.with(|runs| runs.borrow().contains_key(&page)) +} + pub(crate) fn unregister_block_generation(base: usize, size: usize) { if base == 0 || size == 0 { return; diff --git a/crates/perry-runtime/src/arena/tests.rs b/crates/perry-runtime/src/arena/tests.rs index 44c13392be..6d29886bfc 100644 --- a/crates/perry-runtime/src/arena/tests.rs +++ b/crates/perry-runtime/src/arena/tests.rs @@ -1546,8 +1546,8 @@ fn deferred_registration_flush_sites() { "expand_promoted_run", "expands a DESCRIBED promoted page into the object list. Every \ caller has already flushed: the four readers/removers do so as \ - their #7624 obligation, `materialize_all_promoted_page_runs` runs \ - immediately after `old_pages_begin_gc_cycle`, and \ + their #7624 obligation, `materialize_promoted_page_runs_for_object` \ + flushes before it expands (#10182), and \ `register_promoted_page_run` is inside the promotion walk covered \ by the entry above", ), diff --git a/crates/perry-runtime/src/arena/tests_promoted_runs.rs b/crates/perry-runtime/src/arena/tests_promoted_runs.rs index 1c5584e8f3..a08f2a942a 100644 --- a/crates/perry-runtime/src/arena/tests_promoted_runs.rs +++ b/crates/perry-runtime/src/arena/tests_promoted_runs.rs @@ -141,10 +141,11 @@ fn a_full_cycle_expands_every_pending_run_before_it_can_sweep() { let pages = register_region_runs(®ion); assert!(pending_promoted_page_runs() > 0); - // What `GcCycleState::new_full` calls. A run's bounds are addresses - // captured at promotion; once the sweep can free objects inside the - // block and `old_free` can refill the holes, those bounds stop being - // object boundaries. + // What full and budgeted cycle constructors called before #10182 (the + // sweep now expands a page only where it reshapes one). A run's bounds + // are addresses captured at promotion; once the sweep can free objects + // inside the block and `old_free` can refill the holes, those bounds + // stop being object boundaries. materialize_all_promoted_page_runs(); assert_eq!(pending_promoted_page_runs(), 0); diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 2a85720828..024b45ac77 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -624,9 +624,11 @@ impl GcCycleState { let trace = GcCycleTrace::new(GcCollectionKind::Full, trigger); let start = Instant::now(); crate::arena::old_pages_begin_gc_cycle(); - // The one constructor that sweeps old-gen, so the one that invalidates - // a promoted run's bounds. See the fn's doc for why no minor needs it. - crate::arena::materialize_all_promoted_page_runs(); + // #10182: promoted page runs are NOT expanded here any more. The sweep + // expands a run only on a page where it is about to invalidate a dead + // header (`PendingOldUnregister::defer`), and a block it reclaims whole + // drops its runs unexpanded (`unregister_old_block_pages`). A page on + // which every object survives keeps its run: nothing reshapes it. clear_mark_seeds(); // Allocate-black for the WHOLE cycle, from the first build slice on: // the mark barrier only engages at the END of BuildValidPointerSet diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs index 74daed90c1..50303dfeb3 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_batch.rs @@ -13,10 +13,23 @@ use super::super::*; /// of a large heap never stages an unbounded buffer. const FLUSH_AT: usize = 4096; -#[derive(Default)] pub(super) struct PendingOldUnregister { dead: Vec<(usize, usize)>, scratch: Vec<(usize, usize, usize)>, + /// `(first page, last page)` of the last dead header whose described + /// promoted runs were expanded: consecutive dead objects on one page expand + /// it once. + expanded_pages: (usize, usize), +} + +impl Default for PendingOldUnregister { + fn default() -> Self { + Self { + dead: Vec::new(), + scratch: Vec::new(), + expanded_pages: (usize::MAX, usize::MAX), + } + } } impl PendingOldUnregister { @@ -26,6 +39,30 @@ impl PendingOldUnregister { /// /// `header` must be a dead old-gen arena header of `total_size` bytes. pub(super) unsafe fn defer(&mut self, header: *mut GcHeader, total_size: usize) { + // #10182: a described promoted page is re-parsed by header type when it + // is expanded, so it must be expanded BEFORE this header stops parsing + // as an object. Expanded after, the list would silently lack the dead + // object, the batched removal below would not find it, and the page's + // `allocated_bytes`/`object_count` would keep counting it. + #[cfg(test)] + let expand = super::super::trace::block_skip::sabotage::get() + & super::super::trace::block_skip::sabotage::FORGET_RUN_EXPANSION + == 0; + #[cfg(not(test))] + let expand = true; + if expand { + let pages = ( + crate::arena::generation_page_for_addr(header as usize), + crate::arena::generation_page_for_addr(header as usize + total_size - 1), + ); + if pages != self.expanded_pages { + crate::arena::materialize_promoted_page_runs_for_object( + header as usize, + total_size, + ); + self.expanded_pages = pages; + } + } (*header).obj_type = 0; (*header).gc_flags = 0; (*header)._reserved = 0; diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index bb3696fe35..ff1ec82d15 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -69,6 +69,7 @@ mod start_bitmap; mod step_bounds; pub(super) mod support; mod survival_diag; +mod sweep_described_runs; mod sweep_page_tally; mod teardown; mod telemetry_verifier; diff --git a/crates/perry-runtime/src/gc/tests/promote_in_place.rs b/crates/perry-runtime/src/gc/tests/promote_in_place.rs index 49d0f12c09..91a5d09cf0 100644 --- a/crates/perry-runtime/src/gc/tests/promote_in_place.rs +++ b/crates/perry-runtime/src/gc/tests/promote_in_place.rs @@ -953,8 +953,9 @@ fn old_page_relocation_expands_a_described_run_before_it_moves_anything() { /// comment: a page that has only just been promoted is not defrag-eligible at /// all, because `register_promoted_page_run` records live bytes and never dead /// ones, and `old_page_defrag_eligible` requires `dead_bytes > 0`. Dead bytes -/// come only from the old-gen sweep, and the sweep's cycle constructor -/// (`GcCycleState::new_full`) expands every pending run before it starts. +/// come only from the old-gen sweep, and since #10182 the sweep expands a +/// page's pending run before it invalidates the first dead header on that page +/// (`PendingOldUnregister::defer`), so no page with dead bytes keeps a run. #[test] fn a_freshly_described_page_is_not_defrag_eligible() { let _isolation = copying_nursery_isolation_lock(); diff --git a/crates/perry-runtime/src/gc/tests/sweep_described_runs.rs b/crates/perry-runtime/src/gc/tests/sweep_described_runs.rs new file mode 100644 index 0000000000..07b107bdcb --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/sweep_described_runs.rs @@ -0,0 +1,221 @@ +//! #10182: a full collection no longer expands every described promoted page +//! run when it starts. The sweep expands a page's run only right before it +//! invalidates a dead header on that page, and a page on which every object +//! survives keeps its run. +//! +//! The population is real old-gen memory whose eager page index is replaced by +//! described runs, the way an untraced in-place promotion leaves a block. Each +//! case runs one synchronous full on a fresh thread and compares the page index +//! and the page accounting against the planted liveness. The sabotaged twin +//! invalidates dead headers without expanding their page first and shows the +//! page accounting keeps counting a freed object. + +use super::super::*; +use super::support::*; +use crate::gc::trace::block_skip::sabotage; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("described-run sweep test thread must not panic"); +} + +struct Planted { + /// `(header, total size)` of every planted object, ascending. + objects: Vec<(usize, usize)>, + /// A page strictly inside the planted span. + page: usize, +} + +impl Planted { + fn overlaps_page(&self, header: usize, size: usize) -> bool { + let page_base = crate::arena::generation_page_base(self.page); + header < page_base + crate::arena::GENERATION_PAGE_SIZE && header + size > page_base + } + + fn on_page(&self) -> Vec<(usize, usize)> { + self.objects + .iter() + .copied() + .filter(|&(h, s)| self.overlaps_page(h, s)) + .collect() + } +} + +/// Allocate old strings over several pages, drop their eager page index and +/// DESCRIBE every page instead, exactly as `finish_in_place_promotion` does on +/// its untraced path. +unsafe fn plant_described() -> Planted { + let mut objects = Vec::new(); + for _ in 0..600 { + let user = crate::arena::arena_alloc_gc_old(56, 8, GC_TYPE_STRING) as usize; + let (header, size) = old_test_header_and_size(user); + objects.push((header as usize, size)); + } + objects.sort_unstable(); + crate::arena::old_arena_page_index_clear_for_tests(); + let mut runs: Vec<(usize, usize, usize, usize, usize)> = Vec::new(); + for &(header, size) in &objects { + let first = crate::arena::generation_page_for_addr(header); + let last = crate::arena::generation_page_for_addr(header + size - 1); + for page in first..=last { + let base = crate::arena::generation_page_base(page); + let overlap = + (header + size).min(base + crate::arena::GENERATION_PAGE_SIZE) - header.max(base); + match runs.last_mut() { + Some(run) if run.0 == page => { + run.2 = header; + run.3 += 1; + run.4 += overlap; + } + _ => runs.push((page, header, header, 1, overlap)), + } + } + } + assert!( + runs.len() >= 5, + "premise: the population spans several pages" + ); + for &(page, first, last, count, bytes) in &runs { + crate::arena::register_promoted_page_run(page, first, last, count, bytes); + } + Planted { + objects, + page: runs[runs.len() / 2].0, + } +} + +fn root_all(headers: &[usize]) -> Vec> { + headers + .iter() + .map(|&header| { + let mut slot = Box::new(string_bits(header + GC_HEADER_SIZE)); + js_gc_register_global_root(&mut *slot as *mut u64 as i64); + slot + }) + .collect() +} + +fn page_headers(page: usize) -> Vec { + let mut pages = crate::fast_hash::new_ptr_hash_set(); + pages.insert(page); + let mut seen = Vec::new(); + crate::arena::old_arena_walk_objects_on_pages(&pages, |h| seen.push(h as usize)); + seen.sort_unstable(); + seen +} + +fn synchronous_full() { + let _ = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); +} + +/// Every object overlapping the page survives: the full leaves the page's run +/// described, and the run still expands to exactly those objects. +#[test] +fn a_full_keeps_the_run_of_a_page_whose_objects_all_survive() { + run_isolated(|| { + let planted = unsafe { plant_described() }; + let live: Vec = planted.on_page().iter().map(|&(h, _)| h).collect(); + let _roots = root_all(&live); + assert!(crate::arena::promoted_page_run_pending(planted.page)); + + synchronous_full(); + + assert!( + planted + .objects + .iter() + .any(|&(h, _)| unsafe { (*(h as *const GcHeader)).obj_type == 0 }), + "premise: the full swept the unrooted objects on the other pages" + ); + assert!( + crate::arena::promoted_page_run_pending(planted.page), + "a page on which nothing died is not reshaped, so its run must stay described" + ); + assert_eq!(page_headers(planted.page), live); + }); +} + +struct OneDead { + objects_removed: usize, + bytes_removed: usize, + victim_bytes: usize, + index: Vec, + live: Vec, +} + +fn one_dead_on_the_page(sabotaged: bool) -> OneDead { + let planted = unsafe { plant_described() }; + let on_page = planted.on_page(); + // Kill one object in the middle of the page; root every other object + // overlapping it. + let victim = on_page[on_page.len() / 2]; + let live: Vec = on_page + .iter() + .map(|&(h, _)| h) + .filter(|&h| h != victim.0) + .collect(); + let _roots = root_all(&live); + let before = crate::arena::old_page_meta_for_tests(planted.page).expect("page meta"); + { + let _sabotage = sabotaged.then(|| sabotage::Guard::arm(sabotage::FORGET_RUN_EXPANSION)); + synchronous_full(); + } + let after = crate::arena::old_page_meta_for_tests(planted.page).expect("page meta"); + assert_eq!( + unsafe { (*(victim.0 as *const GcHeader)).obj_type }, + 0, + "premise: the unrooted object was swept" + ); + assert!(!crate::arena::promoted_page_run_pending(planted.page)); + OneDead { + objects_removed: before.object_count - after.object_count, + bytes_removed: before.allocated_bytes - after.allocated_bytes, + victim_bytes: victim.1, + index: page_headers(planted.page), + live, + } +} + +/// One object on the page dies: the sweep expands the page before it +/// invalidates the header, so the batched removal finds the object and the +/// page's accounting drops exactly that object. +#[test] +fn a_dead_object_on_a_described_page_leaves_the_page_accounting_exact() { + run_isolated(|| { + let r = one_dead_on_the_page(false); + assert_eq!( + r.objects_removed, 1, + "the page must stop counting the freed object" + ); + assert_eq!(r.bytes_removed, r.victim_bytes, "and exactly its bytes"); + assert_eq!( + r.index, r.live, + "the page index holds exactly the survivors" + ); + }); +} + +#[test] +fn sabotaged_expansion_order_keeps_counting_a_freed_object() { + run_isolated(|| { + let r = one_dead_on_the_page(true); + assert_eq!( + r.index, r.live, + "the index still reads right: the harm is silent" + ); + assert_eq!( + r.objects_removed, 0, + "expanded after the header was invalidated, the run no longer lists the \ + dead object, the removal misses it, and the page keeps counting it" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/trace/block_skip.rs b/crates/perry-runtime/src/gc/trace/block_skip.rs index dee36a19bd..6cbf02fc1f 100644 --- a/crates/perry-runtime/src/gc/trace/block_skip.rs +++ b/crates/perry-runtime/src/gc/trace/block_skip.rs @@ -276,6 +276,9 @@ pub(crate) mod sabotage { /// The sweep never applies its per-page accounting tally before a page-index /// flush or a step end (it is applied only at page changes). pub(crate) const FORGET_PAGE_TALLY_ORDER: u8 = 8; + /// A dead old header is invalidated without first expanding the described + /// promoted run of its page. + pub(crate) const FORGET_RUN_EXPANSION: u8 = 16; thread_local! { static SABOTAGE: Cell = const { Cell::new(0) }; From 7de1da1a2473b159120ae2aa7394f53613fe6c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 23:44:31 +0200 Subject: [PATCH 13/40] gc: an unbudgeted census walks each arena block in one pass (#10182) The exact census reads every header in the arena. On a pacing full over two promoted 20 MB JSON trees it spent ~15 ms doing it, against ~4 ms for a plain header walk over the same 1.8 M objects: the cost was the per-object cursor call and the per-object recomputation of block constants, not the memory traffic. ValidPointerSetBuilder::census_whole_block parses a block itself (same alignment, stop conditions and walkability filter as next_budgeted), sets start bits through the block's chunk pointers, and applies the pointer range, start count and nursery classification once per block. Budgeted steps and classifier-mode sets keep the per-object walk. --- crates/perry-runtime/src/arena/walk.rs | 21 ++ .../src/gc/tests/census_whole_block.rs | 194 ++++++++++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + crates/perry-runtime/src/gc/trace.rs | 169 ++++++++++++++- 4 files changed, 384 insertions(+), 1 deletion(-) create mode 100644 crates/perry-runtime/src/gc/tests/census_whole_block.rs diff --git a/crates/perry-runtime/src/arena/walk.rs b/crates/perry-runtime/src/arena/walk.rs index 2031b6c9c1..e1e7f34fdf 100644 --- a/crates/perry-runtime/src/arena/walk.rs +++ b/crates/perry-runtime/src/arena/walk.rs @@ -268,6 +268,27 @@ impl ArenaObjectCursor { self.finished } + /// Hand out the next whole block as `(global block index, data, offset, + /// size)`, snapshotted exactly as `next_budgeted` would walk it, for a + /// caller that parses the block itself (#10182). Honours `set_skip_blocks`. + /// Only valid at a block boundary ([`Self::at_block_boundary`]); `None` + /// once the cursor is exhausted. + pub(crate) fn next_whole_block(&mut self) -> Option<(usize, usize, usize, usize)> { + debug_assert!(self.at_block_boundary()); + if !self.ensure_current_block() { + return None; + } + let block = self.current_block.take()?; + self.offset = 0; + Some((block.block_idx, block.data, block.offset, block.size)) + } + + /// No block has been entered by `next_budgeted` (the start of the walk, or + /// the point right after a block was exhausted). + pub(crate) fn at_block_boundary(&self) -> bool { + self.current_block.is_none() && self.offset == 0 + } + /// Never enter the blocks whose global index is set in `skip` (#10182). /// Must be installed before the first `next`; a block the cursor is /// already inside is not affected. diff --git a/crates/perry-runtime/src/gc/tests/census_whole_block.rs b/crates/perry-runtime/src/gc/tests/census_whole_block.rs new file mode 100644 index 0000000000..a8b7e9d018 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/census_whole_block.rs @@ -0,0 +1,194 @@ +//! #10182: an unbudgeted census walks each arena block in one pass +//! (`ValidPointerSetBuilder::census_whole_block`) instead of one cursor call +//! per object. It must build exactly the set the per-object walk builds. +//! +//! The population mixes everything the census records per object: plain +//! strings of odd sizes across bitmap words, an oversized block (sorted start +//! list), nursery objects stamped tenured (tenured-nursery bytes), and every +//! per-object obligation the block-skip sweep reads — a promise and a Set +//! (finalize hooks), a pinned object, an already-marked header, an array with +//! raw-f64 layout bits — plus an invalidated header the walk must step over. +//! A census built with a small work budget takes the per-object path; the +//! unbudgeted one takes the whole-block path; every recorded fact is compared. +//! The sabotaged twin drops the whole-block walk's start bits. + +use super::super::*; +use super::support::*; +use crate::gc::trace::whole_block_census_sabotage; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("whole-block census test thread must not panic"); +} + +struct Population { + marked: usize, +} + +unsafe fn plant() -> Population { + for i in 0..6000usize { + let payload = [0usize, 8, 13, 40, 200][i % 5]; + crate::arena::arena_alloc_gc_old(payload, 8, GC_TYPE_STRING); + } + let _promise = alloc_old_test_promise(); + let (_set, _elements, _layout) = alloc_old_test_set(4); + let pinned = crate::arena::arena_alloc_gc_old(24, 8, GC_TYPE_STRING) as usize; + crate::gc::pin_object(header_from_user_ptr(pinned as *const u8) as *mut GcHeader); + let marked = crate::arena::arena_alloc_gc_old(24, 8, GC_TYPE_STRING) as usize; + (*(header_from_user_ptr(marked as *const u8) as *mut GcHeader)).gc_flags |= GC_FLAG_MARKED; + let (raw_array, _) = alloc_old_test_array(4); + (*(header_from_user_ptr(raw_array as *const u8) as *mut GcHeader))._reserved |= + GC_ARRAY_RAW_F64_LAYOUT; + let hole = crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_STRING) as usize; + (*(header_from_user_ptr(hole as *const u8) as *mut GcHeader)).obj_type = 0; + for _ in 0..3000usize { + crate::arena::arena_alloc_gc_old(16, 8, GC_TYPE_STRING); + } + crate::arena::arena_alloc_gc_old(2 * crate::arena::BLOCK_SIZE + 4096, 8, GC_TYPE_STRING); + for i in 0..2000usize { + let user = young_leaf(); + if i % 3 == 0 { + (*(header_from_user_ptr(user as *const u8) as *mut GcHeader)).gc_flags |= + GC_FLAG_TENURED; + } + alloc_nursery_test_object(2); + } + Population { marked } +} + +fn stepped_census() -> ValidPointerSet { + let mut builder = ValidPointerSetBuilder::new(); + while !builder.step(7) {} + builder.finish() +} + +#[derive(Debug, Default)] +struct Diff { + blocks_compared: usize, + sorted_blocks: usize, + obligation_blocks: usize, + mismatches: Vec, +} + +fn compare(stepped: &ValidPointerSet, whole: &ValidPointerSet) -> Diff { + let mut diff = Diff::default(); + let mut note = |what: String| diff.mismatches.push(what); + if stepped.arena_count != whole.arena_count { + note(format!( + "arena_count {} vs {}", + stepped.arena_count, whole.arena_count + )); + } + if (stepped.range_min, stepped.range_max) != (whole.range_min, whole.range_max) { + note("pointer range".into()); + } + if stepped.tenured_nursery_bytes() != whole.tenured_nursery_bytes() { + note(format!( + "tenured nursery bytes {} vs {}", + stepped.tenured_nursery_bytes(), + whole.tenured_nursery_bytes() + )); + } + if stepped.arena_block_bases != whole.arena_block_bases { + note("block fences".into()); + } + if stepped.start_bitmap_chunks != whole.start_bitmap_chunks { + note("start bitmap contents".into()); + } + if stepped.large_starts != whole.large_starts { + note("sorted start lists".into()); + } + for (a, b) in stepped.arena_blocks.iter().zip(&whole.arena_blocks) { + if (a.base, a.extent, a.block_idx, a.first, a.len, a.sorted) + != (b.base, b.extent, b.block_idx, b.first, b.len, b.sorted) + { + note(format!("census block {:#x}", a.base)); + } + } + for block_idx in 0..crate::arena::arena_block_count() { + let (a, b) = ( + stepped.block_census.block(block_idx), + whole.block_census.block(block_idx), + ); + match (a, b) { + (None, None) => {} + (Some(a), Some(b)) => { + diff.blocks_compared += 1; + if a.obligation { + diff.obligation_blocks += 1; + } + if (a.data, a.end, a.objects, a.bytes, a.obligation, a.premarked) + != (b.data, b.end, b.objects, b.bytes, b.obligation, b.premarked) + { + diff.mismatches + .push(format!("block facts {block_idx}: {a:?} vs {b:?}")); + } + } + _ => diff + .mismatches + .push(format!("block {block_idx} censused by one walk only")), + } + } + diff.sorted_blocks = whole.arena_blocks.iter().filter(|b| b.sorted).count(); + diff +} + +#[test] +fn the_whole_block_census_records_exactly_what_the_per_object_census_records() { + run_isolated(|| { + let population = unsafe { plant() }; + let stepped = stepped_census(); + let whole = ValidPointerSetBuilder::new().finish(); + let diff = compare(&stepped, &whole); + assert!( + diff.blocks_compared >= 4, + "premise: several census blocks: {diff:?}" + ); + assert!( + diff.sorted_blocks >= 1, + "premise: an oversized block: {diff:?}" + ); + assert!( + diff.obligation_blocks >= 1, + "premise: the obligation objects were censused: {diff:?}" + ); + assert!( + stepped.tenured_nursery_bytes() > 0, + "premise: tenured nursery bytes were recorded" + ); + assert!(diff.mismatches.is_empty(), "{:?}", diff.mismatches); + unsafe { + (*(header_from_user_ptr(population.marked as *const u8) as *mut GcHeader)).gc_flags &= + !GC_FLAG_MARKED; + } + }); +} + +#[test] +fn sabotaged_whole_block_census_is_caught_by_the_comparison() { + run_isolated(|| { + let population = unsafe { plant() }; + let stepped = stepped_census(); + let whole = { + let _sabotage = whole_block_census_sabotage::Guard::arm(); + ValidPointerSetBuilder::new().finish() + }; + let diff = compare(&stepped, &whole); + assert!( + diff.mismatches.iter().any(|m| m.contains("bitmap")), + "a whole-block walk that drops start bits must disagree: {:?}", + diff.mismatches + ); + unsafe { + (*(header_from_user_ptr(population.marked as *const u8) as *mut GcHeader)).gc_flags &= + !GC_FLAG_MARKED; + } + }); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index ff1ec82d15..0b496ae00f 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -11,6 +11,7 @@ mod budgeted_step_api; mod buffer_bound_method_name; mod buffer_side_tables; mod census; +mod census_whole_block; mod concat_site; mod contract; mod copying; diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index f8e75ee301..65835f4bc7 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -364,7 +364,7 @@ impl ValidPointerSet { } #[inline(always)] - fn record_pointer_range(&mut self, ptr: usize) { + pub(super) fn record_pointer_range(&mut self, ptr: usize) { if ptr < self.range_min { self.range_min = ptr; } @@ -559,6 +559,37 @@ impl ValidPointerSet { } } +/// Sabotage switch for the whole-block census test: the one-pass walk skips +/// recording every start bit. Test builds only. +#[cfg(test)] +pub(crate) mod whole_block_census_sabotage { + use std::cell::Cell; + + thread_local! { + static DROP_STARTS: Cell = const { Cell::new(false) }; + } + + #[inline] + pub(crate) fn dropping_starts() -> bool { + DROP_STARTS.with(Cell::get) + } + + /// Arms the dropped starts until the guard drops. + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(DROP_STARTS.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + DROP_STARTS.with(|s| s.set(self.0)); + } + } +} + /// Sabotage switch for the start-bitmap tests: shifts every bitmap probe by /// one alignment unit, so a test can show its membership oracle notices a /// bitmap that answers for the wrong address. Test builds only. @@ -692,6 +723,10 @@ impl ValidPointerSetBuilder { } } ValidPointerSetBuildPhase::ArenaWalk => { + if unbounded && self.walk_whole_blocks() { + self.phase = ValidPointerSetBuildPhase::MallocWalk; + continue; + } if !self.step_arena_walk(&mut remaining) { return false; } @@ -780,6 +815,138 @@ impl ValidPointerSetBuilder { false } + /// An unbudgeted census walks each block in one pass + /// ([`Self::census_whole_block`]) instead of one cursor call per object: + /// the census reads every header of the arena, and on a pacing full over + /// two promoted 20 MB JSON trees its per-object overhead, not the memory + /// traffic, was three quarters of the phase (#10182). Returns false, and + /// walks nothing, when the per-object path must be kept: classifier mode + /// records no starts, and a cursor a budgeted step left inside a block + /// resumes mid-block. + fn walk_whole_blocks(&mut self) -> bool { + if self.set.classifier_mode + || !self + .arena_cursor + .as_ref() + .is_some_and(crate::arena::ArenaObjectCursor::at_block_boundary) + { + return false; + } + loop { + let next = self + .arena_cursor + .as_mut() + .expect("arena cursor exists during arena walk") + .next_whole_block(); + let Some((block_idx, data, offset, size)) = next else { + self.arena_cursor = None; + return true; + }; + // SAFETY: the cursor snapshotted this block for this census, and + // nothing runs between that snapshot and this walk. + unsafe { self.census_whole_block(block_idx, data, offset, size) }; + } + } + + /// Census one whole arena block in a single pass. It yields exactly the + /// headers `ArenaObjectCursor::next_budgeted` yields for the block (same + /// alignment, stop conditions and walkability filter) and records exactly + /// what `step_arena_walk` records for each of them; the per-block constants + /// (the bitmap chunks, the nursery classification, the pointer range) are + /// hoisted out of the per-object loop. + /// + /// # Safety + /// `data`/`offset`/`size` are an arena block as the census cursor + /// snapshotted it. + unsafe fn census_whole_block( + &mut self, + block_idx: usize, + data: usize, + offset: usize, + size: usize, + ) { + let mut cursor = 0usize; + let mut begun = false; + let mut bitmap: Option<[*mut u64; CENSUS_BITMAP_MAX_CHUNKS]> = None; + let mut nursery = false; + let mut first_start = 0usize; + let mut last_start = 0usize; + let mut bitmap_starts = 0usize; + let mut tenured_bytes = 0usize; + while cursor < offset { + let aligned = (cursor + 7) & !7; + if aligned >= offset { + break; + } + let header = (data + aligned) as *const GcHeader; + let total_size = (*header).size as usize; + if total_size == 0 || total_size > size { + break; + } + cursor = aligned + total_size; + if !crate::gc::gc_type_is_arena_walkable((*header).obj_type) { + continue; + } + let user_ptr = data + aligned + GC_HEADER_SIZE; + if !begun { + begun = true; + self.census_block_idx = block_idx; + self.set.begin_arena_block( + u32::try_from(block_idx).unwrap_or(u32::MAX), + data, + offset, + ); + if self.census_armed { + self.set.block_census.begin_block(block_idx, data, offset); + } + let block = self + .set + .arena_blocks + .last() + .expect("census block just opened"); + if !block.sorted { + bitmap = Some(block.chunks); + } + // Every object of the block has the block's classification. + nursery = crate::arena::pointer_in_nursery(user_ptr); + first_start = user_ptr; + } + if self.census_armed { + self.set.block_census.note_header(header); + } + match bitmap { + Some(chunks) => { + #[cfg(test)] + if whole_block_census_sabotage::dropping_starts() { + last_start = user_ptr; + continue; + } + // `aligned` is a multiple of 8 below `offset`, the block's + // extent: the bit and its word are inside the bitmap. + let bit = aligned >> CENSUS_START_ALIGN_SHIFT; + let word = bit >> 6; + *chunks[word >> CENSUS_BITMAP_CHUNK_WORD_SHIFT] + .add(word & (CENSUS_BITMAP_CHUNK_WORDS - 1)) |= 1u64 << (bit & 63); + bitmap_starts += 1; + } + None => self.set.push_arena(user_ptr), + } + last_start = user_ptr; + let flags = (*header).gc_flags; + if nursery && flags & GC_FLAG_TENURED != 0 && flags & GC_FLAG_FORWARDED == 0 { + tenured_bytes += total_size; + } + } + if begun { + if bitmap.is_some() { + self.set.arena_count += bitmap_starts; + self.set.record_pointer_range(first_start); + self.set.record_pointer_range(last_start); + } + self.set.record_tenured_nursery_bytes(tenured_bytes); + } + } + fn record_arena_header(&mut self, header_ptr: *mut u8) { let user_ptr = unsafe { header_ptr.add(GC_HEADER_SIZE) }; self.set.push_arena(user_ptr as usize); From dd7f353435844ec634130c65a5cfdab187a67446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 23:47:00 +0200 Subject: [PATCH 14/40] gc: a full's hole-list rebuild skips live blocks the census proved hole-free (#10182) After the object walk, old_free_rebuild_from_live_old_blocks re-parsed every live old block looking for invalidated headers: on a pacing full that keeps one promoted 20 MB JSON tree, a second pass over the whole tree that finds no hole. The whole-block census now records whether a block holds any header that does not parse as an object, and the sweep records where it invalidated one; the rebuild skips a live old block that is hole-free by both, unchanged since the census. Counter: hole_rebuild_blocks_skipped; diag [gc-old-free] rebuild_skipped_blocks=. --- .../src/gc/oldgen/sweep_objects.rs | 77 +++++++++- crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/sweep_hole_rebuild.rs | 133 ++++++++++++++++++ crates/perry-runtime/src/gc/trace.rs | 10 ++ .../perry-runtime/src/gc/trace/block_skip.rs | 44 ++++++ 5 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/sweep_hole_rebuild.rs diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs index 0cd37bc9ab..c91dd896a3 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs @@ -60,6 +60,12 @@ pub(super) struct ArenaSweepObjectsState { block_skip_blocks: u64, block_skip_objects: u64, block_skip_bytes: u64, + /// #10182: per block, the census parsed every header and none of them was + /// invalidated, and the block has not changed since. Empty unless the + /// block skip ran against an armed census. + census_hole_free: Vec, + /// #10182: per block, this sweep invalidated a dead header in it. + invalidated_in_block: Vec, } impl ArenaSweepObjectsState { @@ -100,6 +106,8 @@ impl ArenaSweepObjectsState { block_skip_blocks: 0, block_skip_objects: 0, block_skip_bytes: 0, + census_hole_free: Vec::new(), + invalidated_in_block: vec![false; n_blocks], } } @@ -130,6 +138,25 @@ impl ArenaSweepObjectsState { return; } let survivors = crate::arena::survivor_block_index_range(); + #[cfg(not(test))] + let forget_holes = false; + #[cfg(test)] + let forget_holes = super::super::trace::block_skip::sabotage::get() + & super::super::trace::block_skip::sabotage::FORGET_HOLES + != 0; + self.census_hole_free = self + .block_snapshots + .iter() + .enumerate() + .map(|(block_idx, snapshot)| { + census.block(block_idx).is_some_and(|block| { + block.whole_walk + && (!block.non_walkable || forget_holes) + && block.data == snapshot.data + && block.end == snapshot.data.saturating_add(snapshot.offset) + }) + }) + .collect(); let mut skip = vec![false; self.block_snapshots.len()]; let mut any = false; for (block_idx, snapshot) in self.block_snapshots.iter().enumerate() { @@ -201,14 +228,45 @@ impl ArenaSweepObjectsState { /// completes — block liveness is final at that point, and the block /// cleanup that follows only touches blocks with NO live object, which /// the rebuild's filter already skips. + /// + /// #10182: the rebuild also skips a live block that provably holds no + /// hole, i.e. no header with `obj_type == 0`. Those headers are produced + /// only by invalidating a dead old object, and consumed only by reuse. + /// A block qualifies when this cycle's census parsed all of its headers + /// and found none that does not parse as an object, the block has not + /// grown since, and this sweep invalidated nothing in it. Nothing else in a + /// synchronous full writes a header between the census and here. On a + /// pacing full that keeps one promoted JSON tree, the rebuild otherwise + /// re-parses the whole tree to find no hole. pub(super) fn push_live_block_holes(&mut self) { if self.reclaim_dead_old_blocks { - super::old_free_rebuild_from_live_old_blocks( - &self.block_has_live, - self.old_block_start, - ); + let mut parse = self.block_has_live.clone(); + let mut skipped = 0u64; + for (block_idx, live) in parse.iter_mut().enumerate() { + if block_idx >= self.old_block_start + && *live + && self + .census_hole_free + .get(block_idx) + .copied() + .unwrap_or(false) + && !self + .invalidated_in_block + .get(block_idx) + .copied() + .unwrap_or(true) + { + *live = false; + skipped += 1; + } + } + super::super::trace::block_skip::note_hole_rebuild_blocks_skipped(skipped); + super::old_free_rebuild_from_live_old_blocks(&parse, self.old_block_start); if crate::gc::gc_diag_enabled() { - eprintln!("[gc-old-free] reusable_bytes={}", super::old_free_bytes()); + eprintln!( + "[gc-old-free] reusable_bytes={} rebuild_skipped_blocks={skipped}", + super::old_free_bytes() + ); } } } @@ -464,6 +522,7 @@ impl ArenaSweepObjectsState { gc_type_clear_dead_payload_side_tables((*header).obj_type, user_ptr as usize); } if self.reclaim_dead_old_blocks && dead_old { + self.note_invalidated(block_idx); self.defer_old_unregister(header, total_size); } else { (*header).gc_flags = flags & !(GC_FLAG_FORWARDED | GC_FLAG_MARKED); @@ -483,9 +542,17 @@ impl ArenaSweepObjectsState { } finalize_dead_arena_payload(header, user_ptr, self.overflow_active); if self.reclaim_dead_old_blocks && dead_old { + self.note_invalidated(block_idx); self.defer_old_unregister(header, total_size); } } + + #[inline] + fn note_invalidated(&mut self, block_idx: usize) { + if let Some(slot) = self.invalidated_in_block.get_mut(block_idx) { + *slot = true; + } + } } /// Test builds re-check every skip against the headers themselves: a block the diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 0b496ae00f..0df8ddef6f 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -71,6 +71,7 @@ mod step_bounds; pub(super) mod support; mod survival_diag; mod sweep_described_runs; +mod sweep_hole_rebuild; mod sweep_page_tally; mod teardown; mod telemetry_verifier; diff --git a/crates/perry-runtime/src/gc/tests/sweep_hole_rebuild.rs b/crates/perry-runtime/src/gc/tests/sweep_hole_rebuild.rs new file mode 100644 index 0000000000..6c5a8b462a --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/sweep_hole_rebuild.rs @@ -0,0 +1,133 @@ +//! #10182: a full's hole-list rebuild (`old_free_rebuild_from_live_old_blocks`) +//! skips a live old block the census proved hole-free: every header parsed, +//! none invalidated, the block unchanged since, and nothing invalidated in it by +//! this sweep. +//! +//! The population fills several old blocks. One interior block keeps every +//! object rooted but carries a pre-existing hole (an invalidated header, the +//! shape a dead object leaves behind in a live block); another is rooted and +//! hole-free. The hole must still reach the free list and the hole-free block +//! must be skipped. The sabotaged twin makes the sweep believe the census saw +//! no hole, and the hole is lost. + +use super::super::*; +use super::support::*; +use crate::gc::trace::block_skip::{hole_rebuild_blocks_skipped, sabotage}; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("hole-rebuild test thread must not panic"); +} + +fn block_base(user: usize) -> usize { + crate::arena::classify_heap_space_in_range(user) + .map(|(_, base, _)| base) + .expect("planted object must be in a registered arena block") +} + +struct Planted { + /// Every planted user pointer except the hole, rooted. + _roots: Vec>, + hole_user: usize, + hole_size: usize, +} + +/// Allocate 4.5 blocks of 64-byte old strings, root all of them, and turn one +/// object in the second block into a hole. Its size is unique in the old arena +/// so the free list can be asked for exactly that hole. +unsafe fn plant() -> Planted { + let mut users = Vec::new(); + let mut bytes = 0usize; + let mut hole_user = 0usize; + while bytes < 4 * crate::arena::BLOCK_SIZE + crate::arena::BLOCK_SIZE / 2 { + let payload = if hole_user == 0 && bytes > crate::arena::BLOCK_SIZE + 4096 { + 200 + } else { + 56 + }; + let user = crate::arena::arena_alloc_gc_old(payload, 8, GC_TYPE_STRING) as usize; + let size = old_test_header_and_size(user).1; + if payload == 200 { + hole_user = user; + } else { + users.push(user); + } + bytes += size; + } + let (hole_header, hole_size) = old_test_header_and_size(hole_user); + assert_ne!( + block_base(hole_user), + block_base(users[0]), + "premise: interior block" + ); + super::super::invalidate_dead_old_arena_header(hole_header, hole_size); + let roots = users + .iter() + .map(|&user| { + let mut slot = Box::new(string_bits(user)); + js_gc_register_global_root(&mut *slot as *mut u64 as i64); + slot + }) + .collect(); + Planted { + _roots: roots, + hole_user, + hole_size, + } +} + +fn synchronous_full() { + let _ = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); +} + +fn hole_listed(planted: &Planted) -> bool { + let taken = super::super::old_free::old_free_take_exact(planted.hole_size, None); + if let Some(user) = taken { + // Put it back so the sweep's accounting is left as found. + super::super::old_free::old_free_push_for_test(user, planted.hole_size); + } + taken == Some(planted.hole_user) +} + +#[test] +fn a_hole_in_a_live_block_reaches_the_free_list_and_hole_free_blocks_are_not_parsed() { + run_isolated(|| { + let planted = unsafe { plant() }; + let skipped_before = hole_rebuild_blocks_skipped(); + + synchronous_full(); + + assert!( + hole_rebuild_blocks_skipped() > skipped_before, + "the rooted hole-free blocks must be skipped by the rebuild" + ); + assert!( + hole_listed(&planted), + "the pre-existing hole in a live block must still be on the free list" + ); + }); +} + +#[test] +fn sabotaged_hole_census_loses_the_hole() { + run_isolated(|| { + let planted = unsafe { plant() }; + { + let _sabotage = sabotage::Guard::arm(sabotage::FORGET_HOLES); + synchronous_full(); + } + assert!( + !hole_listed(&planted), + "a block believed hole-free is not parsed, so its hole never reaches the list" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 65835f4bc7..6dd1e804fa 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -873,6 +873,7 @@ impl ValidPointerSetBuilder { let mut last_start = 0usize; let mut bitmap_starts = 0usize; let mut tenured_bytes = 0usize; + let mut non_walkable_before_first_object = false; while cursor < offset { let aligned = (cursor + 7) & !7; if aligned >= offset { @@ -885,6 +886,11 @@ impl ValidPointerSetBuilder { } cursor = aligned + total_size; if !crate::gc::gc_type_is_arena_walkable((*header).obj_type) { + if begun { + self.set.block_census.note_non_walkable(); + } else { + non_walkable_before_first_object = true; + } continue; } let user_ptr = data + aligned + GC_HEADER_SIZE; @@ -898,6 +904,10 @@ impl ValidPointerSetBuilder { ); if self.census_armed { self.set.block_census.begin_block(block_idx, data, offset); + self.set.block_census.note_whole_block_walk(); + if non_walkable_before_first_object { + self.set.block_census.note_non_walkable(); + } } let block = self .set diff --git a/crates/perry-runtime/src/gc/trace/block_skip.rs b/crates/perry-runtime/src/gc/trace/block_skip.rs index 6cbf02fc1f..3604c32556 100644 --- a/crates/perry-runtime/src/gc/trace/block_skip.rs +++ b/crates/perry-runtime/src/gc/trace/block_skip.rs @@ -66,6 +66,13 @@ pub(crate) struct CensusBlock { /// read it (a subset of `obligation`, kept apart for /// `young_generation_unmarked`). pub(crate) premarked: bool, + /// The census parsed every header of the block itself, walkable or not + /// (`ValidPointerSetBuilder::census_whole_block`), so `non_walkable` is a + /// complete answer. The per-object census never sees a non-walkable header. + pub(crate) whole_walk: bool, + /// Some header in the block does not parse as an arena object — an + /// invalidated dead header (`obj_type == 0`) among them. + pub(crate) non_walkable: bool, } /// Per-block census facts and trace reachability for one cycle's @@ -140,9 +147,27 @@ impl BlockCensus { censused: true, obligation: false, premarked: false, + whole_walk: false, + non_walkable: false, }; } + /// The block just begun is being parsed header by header in one pass. + #[inline] + pub(crate) fn note_whole_block_walk(&mut self) { + if self.armed { + self.current.whole_walk = true; + } + } + + /// The current block holds a header that does not parse as an object. + #[inline] + pub(crate) fn note_non_walkable(&mut self) { + if self.armed { + self.current.non_walkable = true; + } + } + /// Record one censused header of the current block. Branch-light: this /// runs once per arena object in every synchronous full's census. /// @@ -279,6 +304,9 @@ pub(crate) mod sabotage { /// A dead old header is invalidated without first expanding the described /// promoted run of its page. pub(crate) const FORGET_RUN_EXPANSION: u8 = 16; + /// The sweep treats every whole-walked block as holding no invalidated + /// header, whatever the census saw. + pub(crate) const FORGET_HOLES: u8 = 32; thread_local! { static SABOTAGE: Cell = const { Cell::new(0) }; @@ -346,6 +374,22 @@ pub(crate) fn type_needs_per_object_sweep(obj_type: u8, object_side_tables_live: } } +crate::perry_thread_local! { + static HOLE_REBUILD_BLOCKS_SKIPPED: Cell = const { Cell::new(0) }; +} + +/// Record live old blocks one sweep's hole-list rebuild did not parse because +/// they provably hold no invalidated header (live-subject counter). +pub(crate) fn note_hole_rebuild_blocks_skipped(blocks: u64) { + HOLE_REBUILD_BLOCKS_SKIPPED.with(|c| c.set(c.get().saturating_add(blocks))); +} + +/// Live old blocks this thread's hole-list rebuilds skipped, since thread start. +#[cfg(test)] +pub(crate) fn hole_rebuild_blocks_skipped() -> u64 { + HOLE_REBUILD_BLOCKS_SKIPPED.with(Cell::get) +} + crate::perry_thread_local! { static BLOCK_SKIP_RECLAIMED_BLOCKS: Cell = const { Cell::new(0) }; static BLOCK_SKIP_RECLAIMED_OBJECTS: Cell = const { Cell::new(0) }; From a25500517a5f98c25e04796efcc93146c1883050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 23:49:37 +0200 Subject: [PATCH 15/40] gc: an unbudgeted sweep walks each arena block in one pass (#10182) A full's sweep called the arena cursor once per object and re-derived the object's block constants (old or general, from-space membership, age bumping) for each one. On a pacing full that keeps one promoted 20 MB JSON tree that was the sweep's largest cost after block skipping. The unbudgeted sweep now parses each block itself, keeps a marked, unpinned, unforwarded object inline with the block constants hoisted, and hands every other header to process_object unchanged. Budgeted sweeps keep the per-object cursor. --- .../src/gc/oldgen/sweep_objects.rs | 85 ++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/sweep_whole_block.rs | 158 ++++++++++++++++++ .../perry-runtime/src/gc/trace/block_skip.rs | 3 + 4 files changed, 247 insertions(+) create mode 100644 crates/perry-runtime/src/gc/tests/sweep_whole_block.rs diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs index c91dd896a3..64b6e2a8f8 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs @@ -274,6 +274,14 @@ impl ArenaSweepObjectsState { pub(super) fn step(&mut self, budget: usize) -> bool { let mut remaining = budget; let mut done = false; + if budget == usize::MAX && self.cursor.at_block_boundary() { + while let Some((block_idx, data, offset, size)) = self.cursor.next_whole_block() { + // SAFETY: the block was snapshotted by this sweep's cursor. + unsafe { self.sweep_whole_block(block_idx, data, offset, size) }; + } + remaining = 0; + done = true; + } while remaining > 0 { let Some((header_ptr, block_idx)) = self.cursor.next() else { done = true; @@ -295,6 +303,83 @@ impl ArenaSweepObjectsState { done } + /// Sweep one whole block in a single pass (#10182). It visits exactly the + /// headers `ArenaObjectCursor::next_budgeted` yields for the block and + /// handles each exactly as `process_object` would. The common case — a + /// marked, unpinned, unforwarded object in a block that does not age-bump — + /// is `keep_live_object` with the per-block constants (old or general, + /// from-space membership, age bumping) hoisted out of the loop; every other + /// header goes through `process_object` unchanged. + /// + /// # Safety + /// `data`/`offset`/`size` are an arena block as this sweep's cursor + /// snapshotted it. + unsafe fn sweep_whole_block( + &mut self, + block_idx: usize, + data: usize, + offset: usize, + size: usize, + ) { + let is_old = block_idx >= self.old_block_start; + let general = block_idx < self.resettable_general_n; + let age_bump = self.do_age_bump && general; + let from_space = crate::arena::block_in_copying_from_space( + block_idx, + self.resettable_general_n, + &self.active_survivor_blocks, + ); + #[cfg(test)] + let record_live = super::super::trace::block_skip::sabotage::get() + & super::super::trace::block_skip::sabotage::FORGET_WHOLE_BLOCK_LIVE + == 0; + #[cfg(not(test))] + let record_live = true; + let mut kept_live = false; + let mut cursor = 0usize; + while cursor < offset { + let aligned = (cursor + 7) & !7; + if aligned >= offset { + break; + } + let header = (data + aligned) as *mut GcHeader; + let total_size = (*header).size as usize; + if total_size == 0 || total_size > size { + break; + } + cursor = aligned + total_size; + if !crate::gc::gc_type_is_arena_walkable((*header).obj_type) { + continue; + } + let flags = (*header).gc_flags; + if age_bump + || flags & (GC_FLAG_MARKED | GC_FLAG_PINNED | GC_FLAG_FORWARDED) != GC_FLAG_MARKED + { + self.process_object(header, block_idx); + continue; + } + if is_old { + self.account_old_object(header, total_size, true, false); + } + kept_live = true; + if general { + self.eden_live_bytes = self.eden_live_bytes.saturating_add(total_size as u64); + } + self.arena_live_bytes = self.arena_live_bytes.saturating_add(total_size as u64); + if from_space { + self.arena_live_from_space_bytes = self + .arena_live_from_space_bytes + .saturating_add(total_size as u64); + } + (*header).gc_flags = flags & !GC_FLAG_MARKED; + } + if kept_live && record_live { + if let Some(slot) = self.block_has_live.get_mut(block_idx) { + *slot = true; + } + } + } + /// Account one swept old object on its page(s), batching single-page /// objects per page. #[inline] diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 0df8ddef6f..6e009fced8 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -73,6 +73,7 @@ mod survival_diag; mod sweep_described_runs; mod sweep_hole_rebuild; mod sweep_page_tally; +mod sweep_whole_block; mod teardown; mod telemetry_verifier; mod temp_roots; diff --git a/crates/perry-runtime/src/gc/tests/sweep_whole_block.rs b/crates/perry-runtime/src/gc/tests/sweep_whole_block.rs new file mode 100644 index 0000000000..6549192246 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/sweep_whole_block.rs @@ -0,0 +1,158 @@ +//! #10182: an unbudgeted sweep walks each arena block in one pass +//! (`ArenaSweepObjectsState::sweep_whole_block`), keeping marked objects inline +//! and handing every other header to `process_object`. +//! +//! The same deterministic population is planted on two fresh threads — live, +//! pinned and dead old objects (the live ones rooted through one old array), +//! live and dead nursery objects — and one full collection runs on each: on one +//! thread in small work steps, so the sweep takes the per-object path, on the +//! other to completion, so it takes the whole-block path. The sweep statistics +//! and every planted header's fate must be identical. The sabotaged twin keeps +//! objects on the fast path without recording their block as live, and the +//! comparison sees the block reclaimed under them. + +use super::super::*; +use super::support::*; +use crate::gc::trace::block_skip::sabotage; + +#[derive(Debug, PartialEq, Eq)] +struct Result { + /// `(freed, eden live, eden dead, arena live, from-space live, reset + /// blocks)` from the sweep. + stats: (u64, u64, u64, u64, u64, usize), + /// `(obj_type, gc_flags)` of every planted header after the collection, in + /// planting order. + fates: Vec<(u8, u8)>, + live_old_survivors: usize, +} + +fn trace_snapshot() -> GcTriggerSnapshot { + GcTriggerSnapshot { + kind: GcTriggerKind::Manual, + steps_before: Some(GcStepSnapshot::current()), + } +} + +unsafe fn plant_and_collect(stepped: bool, sabotaged: bool) -> Result { + const LIVE_SLOTS: u32 = 32_000; + let (holder, elements) = alloc_old_test_array(LIVE_SLOTS); + let mut root = ptr_bits(holder as usize); + js_gc_register_global_root(&mut root as *mut u64 as i64); + let mut headers = vec![holder as usize - GC_HEADER_SIZE]; + let mut live_slot = 0u32; + let mut state = 0x2545_f491_4f6c_dd1du64; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as usize + }; + let mut live_old = Vec::new(); + for i in 0..60_000usize { + let roll = next(); + let young = i % 5 == 0; + let user = if young { + young_leaf() + } else { + crate::arena::arena_alloc_gc_old(8 + roll % 120, 8, GC_TYPE_STRING) as usize + }; + headers.push(user - GC_HEADER_SIZE); + // The first 40 000 objects (about three blocks) hold survivors and + // garbage but nothing pinned, so whole blocks keep their survivors on + // the fast path alone; the rest add pinned objects. + let pinning_stretch = i >= 40_000; + match roll % 16 { + 0..=9 if live_slot < LIVE_SLOTS && !pinning_stretch => { + *elements.add(live_slot as usize) = string_bits(user); + live_slot += 1; + if !young { + live_old.push(user); + } + } + 10 if !young && pinning_stretch => { + crate::gc::pin_object(header_from_user_ptr(user as *const u8)) + } + _ => {} + } + } + let _sabotage = sabotaged.then(|| sabotage::Guard::arm(sabotage::FORGET_WHOLE_BLOCK_LIVE)); + let mut cycle = GcCycleState::new_full(trace_snapshot()); + let outcome = if stepped { + while !cycle.step(GcWorkBudget::bounded(64)).completed {} + cycle + .take_outcome() + .expect("completed cycle has an outcome") + } else { + cycle.run_to_completion() + }; + let sweep = outcome.trace.expect("trace requested").sweep; + let fates = headers + .iter() + .map(|&h| { + let header = h as *const GcHeader; + ((*header).obj_type, (*header).gc_flags) + }) + .collect(); + let live_old_survivors = live_old + .iter() + .filter(|&&u| crate::arena::pointer_in_old_gen(u)) + .count(); + Result { + stats: ( + sweep.freed_bytes, + sweep.eden_live_bytes, + sweep.eden_dead_bytes, + sweep.arena_live_bytes, + sweep.arena_live_from_space_bytes, + sweep.reset_blocks, + ), + fates, + live_old_survivors, + } +} + +fn on_fresh_thread(stepped: bool, sabotaged: bool) -> Result { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + unsafe { plant_and_collect(stepped, sabotaged) } + }) + .join() + .expect("whole-block sweep test thread must not panic") +} + +#[test] +fn the_whole_block_sweep_matches_the_per_object_sweep() { + let per_object = on_fresh_thread(true, false); + let whole_block = on_fresh_thread(false, false); + assert!( + per_object.stats.0 > 0 && per_object.stats.3 > 0, + "premise: the sweep both freed and kept: {:?}", + per_object.stats + ); + assert!(per_object.live_old_survivors > 0, "premise: old survivors"); + assert_eq!(per_object.stats, whole_block.stats); + assert_eq!( + per_object.live_old_survivors, + whole_block.live_old_survivors + ); + assert!( + per_object.fates == whole_block.fates, + "a planted header's fate differs" + ); +} + +#[test] +fn sabotaged_whole_block_liveness_is_caught_by_the_comparison() { + let per_object = on_fresh_thread(true, false); + let sabotaged = on_fresh_thread(false, true); + assert!( + sabotaged.live_old_survivors < per_object.live_old_survivors, + "keeping objects without marking their block live must release rooted \ + objects' blocks: {} vs {}", + sabotaged.live_old_survivors, + per_object.live_old_survivors + ); +} diff --git a/crates/perry-runtime/src/gc/trace/block_skip.rs b/crates/perry-runtime/src/gc/trace/block_skip.rs index 3604c32556..aab346fe24 100644 --- a/crates/perry-runtime/src/gc/trace/block_skip.rs +++ b/crates/perry-runtime/src/gc/trace/block_skip.rs @@ -307,6 +307,9 @@ pub(crate) mod sabotage { /// The sweep treats every whole-walked block as holding no invalidated /// header, whatever the census saw. pub(crate) const FORGET_HOLES: u8 = 32; + /// The whole-block sweep's fast path keeps objects without recording that + /// their block holds a live object. + pub(crate) const FORGET_WHOLE_BLOCK_LIVE: u8 = 64; thread_local! { static SABOTAGE: Cell = const { Cell::new(0) }; From ab4c3b6a8fb94aa8cbc9fde7afac263b81d66444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 23:51:46 +0200 Subject: [PATCH 16/40] gc: read the mark's per-slot facts once per object (#10182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full mark paid, for every traced slot: a page-generation lookup to build GcMutableSlot::external (read only by the copying minor), a thread-local read for the proxy-observation flag, a header re-read to rule out weak-holder classes, a dynamic callback per slot of a range descriptor, and — because the optimizer hoisted it above the arm check — a thread-local address fetch for the layout-scan counters. The slot's generation is now classified when asked, the proxy and weak-holder facts are read once per object, range descriptors are walked inline, and the layout counter's thread-local lives out of line behind the process-wide arm flag. classifier_verify_enabled tests its cached static before its thread-local. --- crates/perry-runtime/src/gc/copying.rs | 2 +- crates/perry-runtime/src/gc/layout.rs | 24 +++-- crates/perry-runtime/src/gc/telemetry.rs | 13 +++ .../src/gc/tests/mark_slot_hoists.rs | 59 ++++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + crates/perry-runtime/src/gc/trace.rs | 93 +++++++++++++++---- crates/perry-runtime/src/weakref.rs | 19 ++++ 7 files changed, 181 insertions(+), 30 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/mark_slot_hoists.rs diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 6bd4e75ca9..8cfd1424cd 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -745,7 +745,7 @@ impl CopyingNurseryCollector { visit_gc_rewrite_slots(header, |slot| unsafe { slot.record_layout_read(); let before = *slot.slot; - self.visit_slot_with_parent(slot.slot, header, slot.external); + self.visit_slot_with_parent(slot.slot, header, slot.external()); changed |= *slot.slot != before; }); if changed { diff --git a/crates/perry-runtime/src/gc/layout.rs b/crates/perry-runtime/src/gc/layout.rs index 7bb3cff12f..97abde37a6 100644 --- a/crates/perry-runtime/src/gc/layout.rs +++ b/crates/perry-runtime/src/gc/layout.rs @@ -1870,21 +1870,27 @@ pub(super) unsafe fn gc_child_slots(header: *mut GcHeader) -> HeapChildSlotItera pub(super) struct GcMutableSlot { pub(super) slot: *mut u64, pub(super) layout_kind: Option, - pub(super) external: bool, } impl GcMutableSlot { #[inline] pub(super) fn new(slot: *mut u64, layout_kind: Option) -> Self { - let external = !matches!( - crate::arena::classify_heap_generation(slot as usize), + Self { slot, layout_kind } + } + + /// Is the slot's own address outside the old generation? + /// + /// #10182: classified when asked, not when the slot is enumerated. The one + /// reader (the copying minor's `scan_object_fields`) asks immediately, so + /// the answer is the same; the full mark, which enumerates every traced + /// slot through this type and never asks, stopped paying a page-generation + /// lookup per slot. + #[inline] + pub(super) fn external(self) -> bool { + !matches!( + crate::arena::classify_heap_generation(self.slot as usize), crate::arena::HeapGeneration::Old - ); - Self { - slot, - layout_kind, - external, - } + ) } #[inline] diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index 5c7bed3eca..a6fbb58678 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -572,6 +572,19 @@ pub(super) fn layout_scan_trace_active() -> bool { #[inline] pub(super) fn record_layout_child_slot_read(kind: HeapChildSlotReadKind) { + // #10182: the process-wide arm flag inline, the thread-local out of line. + // Inlined together into the mark's per-slot path, the optimizer fetched + // the thread-local's address before testing the flag, i.e. on every + // traced slot of every collection with tracing off. + if !LAYOUT_SCAN_TRACE_ARMED_ANY.load(std::sync::atomic::Ordering::Acquire) { + return; + } + record_layout_child_slot_read_armed(kind); +} + +#[cold] +#[inline(never)] +fn record_layout_child_slot_read_armed(kind: HeapChildSlotReadKind) { if !layout_scan_trace_active() { return; } diff --git a/crates/perry-runtime/src/gc/tests/mark_slot_hoists.rs b/crates/perry-runtime/src/gc/tests/mark_slot_hoists.rs new file mode 100644 index 0000000000..b71a947716 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/mark_slot_hoists.rs @@ -0,0 +1,59 @@ +//! #10182: the full mark reads two facts once per traced object instead of once +//! per slot: whether the proxy registry observes the trace, and whether the +//! object is a weak holder whose weak slots are skipped. +//! +//! The weak-holder fact is the one whose loss would change liveness, so it is +//! pinned with a real collection: a rooted `WeakRef` whose target has no other +//! reference must be cleared by a full. The sabotaged twin makes the per-object +//! fact read false, the weak slot is traced strongly, and the target survives. + +use super::super::*; +use super::support::*; +use crate::gc::trace::mark_hoist_sabotage; + +fn weak_target_cleared(sabotaged: bool) -> bool { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + let target = unsafe { alloc_old_test_object(0).0 as usize }; + let holder = crate::weakref::js_weakref_new(f64::from_bits(ptr_bits(target))); + let mut root = ptr_bits(holder as usize); + js_gc_register_global_root(&mut root as *mut u64 as i64); + assert!( + unsafe { + crate::weakref::is_weak_holder_header( + header_from_user_ptr(holder as *const u8) as *mut GcHeader + ) + }, + "premise: a WeakRef is a weak holder" + ); + { + let _sabotage = sabotaged.then(mark_hoist_sabotage::Guard::arm); + let _ = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); + } + crate::weakref::js_weakref_deref(f64::from_bits(root)).to_bits() + == crate::value::TAG_UNDEFINED + }) + .join() + .expect("mark-hoist test thread must not panic") +} + +#[test] +fn a_full_skips_a_weak_holders_weak_slot_through_the_per_object_fact() { + assert!( + weak_target_cleared(false), + "the target reachable only through the WeakRef's weak slot must be cleared" + ); +} + +#[test] +fn sabotaged_weak_holder_fact_keeps_the_weak_target_alive() { + assert!( + !weak_target_cleared(true), + "with the per-object fact forgotten the weak slot is traced strongly" + ); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 6e009fced8..feb8ebf817 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -48,6 +48,7 @@ mod layout_residue_histogram; mod layout_trace; mod lazy_intrinsic_towers; mod lazy_tape_side_alloc; +mod mark_slot_hoists; mod oldgen; mod os_tag; mod promote_in_place; diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 6dd1e804fa..c42a5cb5a4 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -82,11 +82,11 @@ pub(super) fn classifier_valid_object_start(addr: usize) -> bool { /// #6179: differential-verification mode for the page-metadata classifier. pub(super) fn classifier_verify_enabled() -> bool { - if CLASSIFIER_VERIFY_SUPPRESSED.with(|c| c.get()) { - return false; - } + // The cached process-wide switch first: this runs on every census hit, and + // the suppression flag is a thread-local (#10182). static CACHED: std::sync::OnceLock = std::sync::OnceLock::new(); *CACHED.get_or_init(|| super::env_flag_enabled("PERRY_GC_VERIFY_CLASSIFIER")) + && !CLASSIFIER_VERIFY_SUPPRESSED.with(|c| c.get()) } crate::perry_thread_local! { @@ -1109,14 +1109,16 @@ pub(super) fn try_mark_raw_root_addr(addr: usize, valid_ptrs: &ValidPointerSet) /// on exact mutable roots. `PERRY_CONSERVATIVE_STACK_SCAN=full` forces the /// legacy path for debugging and makes copied-minor ineligible. +#[inline(always)] pub(super) unsafe fn mark_field_into_worklist( val_bits: u64, valid_ptrs: &ValidPointerSet, worklist: &mut Vec<*mut GcHeader>, + proxy_trace_active: bool, ) -> bool { - if crate::proxy::gc_full_trace_active() - && crate::proxy::gc_observe_traced_value(val_bits, valid_ptrs) - { + // `proxy_trace_active` is `crate::proxy::gc_full_trace_active()`, read by + // the caller once for the whole object being traced (#10182). + if proxy_trace_active && crate::proxy::gc_observe_traced_value(val_bits, valid_ptrs) { return false; } let tag = val_bits & TAG_MASK; @@ -1486,28 +1488,79 @@ pub(super) unsafe fn trace_heap_rewrite_slots( valid_ptrs: &ValidPointerSet, worklist: &mut Vec<*mut GcHeader>, ) { + // #10182: two per-object facts read once instead of once per slot — + // whether the proxy registry observes this trace (it changes only when a + // proxy is created, and none is created inside one object's visit), and + // whether the object is one of the weak-holder classes whose weak slots + // the trace skips (its class cannot change while it is traced). Range + // descriptors are walked here directly rather than through a per-slot + // dynamic callback. + let proxy_trace_active = crate::proxy::gc_full_trace_active(); + #[cfg(not(test))] + let weak_holder = crate::weakref::is_weak_holder_header(header); + #[cfg(test)] + let weak_holder = + crate::weakref::is_weak_holder_header(header) && !mark_hoist_sabotage::forgetting_weak(); visit_gc_rewrite_slot_descriptors(header, |descriptor| unsafe { - if let GcMutableSlotDescriptor::PointerFreeRange(range) = descriptor { - if crate::proxy::gc_full_trace_active() { - for i in 0..range.slot_count() { - crate::proxy::gc_observe_traced_value(*range.slot(i), valid_ptrs); - } - } - return; - } - descriptor.visit_slots(&mut |slot| { - if crate::weakref::is_weak_target_trace_slot(header, slot.slot) { + let mut visit_slot = |slot: *mut u64, layout_kind: Option| { + if weak_holder && crate::weakref::is_weak_target_trace_slot(header, slot) { return; } - slot.record_layout_read(); - if slot.layout_kind.is_some() { + if let Some(kind) = layout_kind { + record_layout_child_slot_read(kind); record_trace_slot_read(); } - mark_field_into_worklist(*slot.slot, valid_ptrs, worklist); - }); + mark_field_into_worklist(*slot, valid_ptrs, worklist, proxy_trace_active); + }; + match descriptor { + GcMutableSlotDescriptor::PointerFreeRange(range) => { + if proxy_trace_active { + for i in 0..range.slot_count() { + crate::proxy::gc_observe_traced_value(*range.slot(i), valid_ptrs); + } + } + } + GcMutableSlotDescriptor::Slot(slot) => visit_slot(slot.slot, slot.layout_kind), + GcMutableSlotDescriptor::Range { range, layout_kind } => { + for i in 0..range.slot_count() { + visit_slot(range.slot(i), layout_kind); + } + } + } }); } +/// Sabotage switch for the mark-hoist test: the per-object weak-holder fact +/// reads false, so a weak holder's weak slots are traced strongly. Test builds +/// only. +#[cfg(test)] +pub(crate) mod mark_hoist_sabotage { + use std::cell::Cell; + + thread_local! { + static FORGET_WEAK: Cell = const { Cell::new(false) }; + } + + #[inline] + pub(crate) fn forgetting_weak() -> bool { + FORGET_WEAK.with(Cell::get) + } + + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(FORGET_WEAK.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + FORGET_WEAK.with(|s| s.set(self.0)); + } + } +} + /// Trace array elements. /// Elements may be NaN-boxed JSValues OR raw I64 pointers (codegen stores raw I64 for /// is_pointer/is_array/is_string typed arrays via js_array_set_jsvalue). diff --git a/crates/perry-runtime/src/weakref.rs b/crates/perry-runtime/src/weakref.rs index f5b2c7fa32..4fffd73f38 100644 --- a/crates/perry-runtime/src/weakref.rs +++ b/crates/perry-runtime/src/weakref.rs @@ -369,6 +369,25 @@ pub(crate) unsafe fn header_may_hold_weak_target_slots(header: *mut crate::gc::G /// True when `slot` is a weak target edge and must not be treated as a /// strong child during mark/remembered-set scans. Rewrite/copy passes should /// still visit these slots so live weak targets get moved addresses repaired. +/// Is `header` an object of a class whose weak slots +/// [`is_weak_target_trace_slot`] can name? When false, that function answers +/// false for every slot of the object (#10182: the trace asks this once per +/// object instead of asking the slot question for every slot). +/// +/// # Safety +/// `header` is null or a readable GC header. +#[inline] +pub(crate) unsafe fn is_weak_holder_header(header: *mut crate::gc::GcHeader) -> bool { + if header.is_null() || (*header).obj_type != crate::gc::GC_TYPE_OBJECT { + return false; + } + let obj = (header as *mut u8).add(crate::gc::GC_HEADER_SIZE) as *mut ObjectHeader; + matches!( + (*obj).class_id, + CLASS_ID_WEAKREF | CLASS_ID_WEAK_ENTRY | CLASS_ID_FINALIZATION_RECORD + ) +} + pub(crate) unsafe fn is_weak_target_trace_slot( header: *mut crate::gc::GcHeader, slot: *mut u64, From 3473b1374f1235c279b801cec03d141014c64d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 23:53:09 +0200 Subject: [PATCH 17/40] gc: find a census block through a direct-mapped window index (#10182) Every membership query of the full mark first binary-searched the census block fences (about 150 on a 20 MB JSON pacing full), and after the per-slot hoists that search was the largest single cost left in the mark. The finished census now also builds a table with one entry per 1 MiB window: the greatest block base at or below the window start and the one base, if any, inside it. Arena blocks are at least 1 MiB and never overlap, so a lookup is a shift, a bounds check and one compare. Sets whose bases share a window, or that span more than 16 GiB, keep the binary search. --- .../src/gc/tests/census_block_windows.rs | 129 +++++++++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + crates/perry-runtime/src/gc/trace.rs | 131 ++++++++++++++++++ 3 files changed, 261 insertions(+) create mode 100644 crates/perry-runtime/src/gc/tests/census_block_windows.rs diff --git a/crates/perry-runtime/src/gc/tests/census_block_windows.rs b/crates/perry-runtime/src/gc/tests/census_block_windows.rs new file mode 100644 index 0000000000..626689beac --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/census_block_windows.rs @@ -0,0 +1,129 @@ +//! #10182: census membership finds a pointer's block through a direct-mapped +//! index of 1 MiB windows instead of a binary search over the block fences. +//! +//! A real census over several old blocks, an oversized block and nursery blocks +//! is queried at every block edge, inside every block, between blocks and far +//! outside the heap, and each answer is compared with the binary search it +//! replaces. The sabotaged twin ignores the base inside a window and the +//! comparison notices. A fabricated set whose bases share a window keeps the +//! binary search. + +use super::super::*; +use super::support::*; +use crate::gc::trace::block_window_sabotage; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("block-window test thread must not panic"); +} + +unsafe fn plant() { + for i in 0..60_000usize { + crate::arena::arena_alloc_gc_old([0usize, 24, 120][i % 3], 8, GC_TYPE_STRING); + } + crate::arena::arena_alloc_gc_old(3 * crate::arena::BLOCK_SIZE, 8, GC_TYPE_STRING); + for _ in 0..20_000usize { + young_leaf(); + } +} + +fn mismatches(valid: &ValidPointerSet) -> (usize, usize) { + let mut queries = 0usize; + let mut wrong = 0usize; + let mut probe = |addr: usize| { + queries += 1; + let (direct, search) = valid.census_block_base_both_ways(addr); + if direct != search { + wrong += 1; + } + }; + let window = 1usize << 20; + probe(0); + probe(usize::MAX); + for block in &valid.arena_blocks { + for addr in [ + block.base.saturating_sub(window), + block.base.saturating_sub(1), + block.base, + block.base + 8, + block.base + block.extent / 2, + block.base + block.extent.saturating_sub(1), + block.base + block.extent, + (block.base >> 20) << 20, + ((block.base >> 20) << 20).saturating_sub(1), + ((block.base >> 20) + 1) << 20, + block.base + window, + block.base + 4 * window, + ] { + probe(addr); + } + let mut addr = block.base.saturating_sub(4096); + while addr < block.base + block.extent + 4096 { + probe(addr); + addr += 4093; + } + } + (queries, wrong) +} + +#[test] +fn the_block_window_index_answers_exactly_like_the_fence_search() { + run_isolated(|| { + unsafe { plant() }; + let valid = ValidPointerSetBuilder::new().finish(); + assert!( + !valid.block_windows.is_empty(), + "premise: the census built a block window index" + ); + assert!( + valid.arena_blocks.len() >= 4, + "premise: several census blocks" + ); + assert!( + valid.arena_blocks.iter().any(|b| b.sorted), + "premise: an oversized block" + ); + let (queries, wrong) = mismatches(&valid); + assert!(queries > 1000, "premise: {queries} queries"); + assert_eq!(wrong, 0, "{wrong} of {queries} lookups disagree"); + }); +} + +#[test] +fn sabotaged_block_window_lookup_is_caught_by_the_comparison() { + run_isolated(|| { + unsafe { plant() }; + let valid = ValidPointerSetBuilder::new().finish(); + let (_, wrong) = { + let _sabotage = block_window_sabotage::Guard::arm(); + mismatches(&valid) + }; + assert!( + wrong > 0, + "ignoring the in-window base must disagree somewhere" + ); + }); +} + +#[test] +fn bases_sharing_a_window_keep_the_fence_search() { + let mut valid = ValidPointerSet::new(); + let base = 0x7000_0000_0000usize; + valid.begin_arena_block(0, base, 4096); + valid.push_arena(base + GC_HEADER_SIZE); + valid.begin_arena_block(1, base + 8192, 4096); + valid.push_arena(base + 8192 + GC_HEADER_SIZE); + valid.build_block_windows(); + assert!(valid.block_windows.is_empty()); + assert_eq!( + valid.census_block_base_both_ways(base + 8192 + 16), + (Some(base + 8192), Some(base + 8192)) + ); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index feb8ebf817..b316078f04 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -11,6 +11,7 @@ mod budgeted_step_api; mod buffer_bound_method_name; mod buffer_side_tables; mod census; +mod census_block_windows; mod census_whole_block; mod concat_site; mod contract; diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index c42a5cb5a4..918d7b62a6 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -117,6 +117,14 @@ const CENSUS_BITMAP_MAX_CHUNKS: usize = 2; /// exactly the headers that cursor yields. const CENSUS_START_ALIGN_SHIFT: u32 = 3; +/// Window of the direct-mapped block index (#10182): one arena block (1 MiB). +/// Distinct arena blocks are at least this large and never overlap, so no two +/// census block bases fall into one window. +const CENSUS_BLOCK_WINDOW_SHIFT: u32 = 20; +/// Largest direct-mapped block index built: 16 GiB of address span, 128 KiB of +/// table. A census spread wider keeps the binary search. +const CENSUS_BLOCK_WINDOW_MAX: usize = 1 << 14; + /// One censused arena block, in address order (#10182). #[derive(Clone, Copy, Debug)] pub(super) struct CensusStartBlock { @@ -186,6 +194,13 @@ pub(crate) struct ValidPointerSet { /// Per-block census facts and trace reachability (#10182). Disarmed /// unless this set was built by the production census walk. pub(super) block_census: BlockCensus, + /// Direct-mapped block index (#10182): one entry per 1 MiB window from + /// `block_windows_lo`, holding the index of the greatest census block whose + /// base is at or below the window start and the index of the census block, + /// at most one, whose base lies inside the window (`u32::MAX` for none). + /// Empty means `census_block_at` binary-searches `arena_block_bases`. + pub(super) block_windows: Vec<(u32, u32)>, + pub(super) block_windows_lo: usize, /// Live count of pushed arena starts, kept so `lookup_count` stays O(1). pub(super) arena_count: usize, /// Exact membership for **malloc-tracked** objects only, which have no @@ -232,6 +247,8 @@ impl ValidPointerSet { start_bitmap_chunks: Vec::new(), large_starts: Vec::new(), block_census: BlockCensus::disarmed(), + block_windows: Vec::new(), + block_windows_lo: 0, arena_count: 0, malloc_lookup: std::collections::BTreeSet::new(), range_min: usize::MAX, @@ -328,6 +345,46 @@ impl ValidPointerSet { self.record_pointer_range(ptr); } + /// Build the direct-mapped block index once the census is complete. It is + /// left empty — `census_block_at` keeps the binary search — when two block + /// bases share a window (only fabricated test sets do) or the blocks span + /// more than `CENSUS_BLOCK_WINDOW_MAX` windows. + pub(super) fn build_block_windows(&mut self) { + self.block_windows.clear(); + let (Some(first), Some(last)) = (self.arena_blocks.first(), self.arena_blocks.last()) + else { + return; + }; + let lo = first.base >> CENSUS_BLOCK_WINDOW_SHIFT; + let hi = last.base.saturating_add(last.extent.max(1) - 1) >> CENSUS_BLOCK_WINDOW_SHIFT; + let windows = hi - lo + 1; + if windows > CENSUS_BLOCK_WINDOW_MAX || self.arena_blocks.len() >= u32::MAX as usize { + return; + } + let mut table = vec![(u32::MAX, u32::MAX); windows]; + let mut previous = usize::MAX; + for (idx, block) in self.arena_blocks.iter().enumerate() { + let window = (block.base >> CENSUS_BLOCK_WINDOW_SHIFT) - lo; + if window == previous { + return; + } + table[window].1 = idx as u32; + previous = window; + } + let mut greatest = u32::MAX; + let mut next = 0usize; + for (window, entry) in table.iter_mut().enumerate() { + let window_start = (lo + window) << CENSUS_BLOCK_WINDOW_SHIFT; + while next < self.arena_blocks.len() && self.arena_blocks[next].base <= window_start { + greatest = next as u32; + next += 1; + } + entry.0 = greatest; + } + self.block_windows = table; + self.block_windows_lo = lo; + } + pub(super) fn push_malloc(&mut self, ptr: usize) { if self.classifier_mode { return; // #6179: no exact census in classifier mode @@ -463,8 +520,41 @@ impl ValidPointerSet { /// The census block whose base is the greatest one `<= ptr`, if any. /// `ptr` may still lie past that block's walked extent. + /// + /// #10182: answered from the direct-mapped block index when one was built. + /// The window holding `ptr` names the greatest base at or below the window + /// start and the one base, if any, inside the window, so the greatest base + /// at or below `ptr` is the inside one when `ptr` has reached it. Past the + /// last window every base is below `ptr`; before the first none is. #[inline(always)] fn census_block_at(&self, ptr: usize) -> Option<&CensusStartBlock> { + if !self.block_windows.is_empty() { + let window = ptr >> CENSUS_BLOCK_WINDOW_SHIFT; + let idx = match self + .block_windows + .get(window.wrapping_sub(self.block_windows_lo)) + { + Some(&(below, inside)) => { + #[cfg(test)] + if block_window_sabotage::ignoring_inside() { + return self.arena_blocks.get(below as usize); + } + if inside != u32::MAX && ptr >= self.arena_block_bases[inside as usize] { + inside + } else { + below + } + } + None if window < self.block_windows_lo => return None, + None => return self.arena_blocks.last(), + }; + return self.arena_blocks.get(idx as usize); + } + self.census_block_at_by_search(ptr) + } + + #[inline(always)] + fn census_block_at_by_search(&self, ptr: usize) -> Option<&CensusStartBlock> { let idx = self.arena_block_bases.partition_point(|&base| base <= ptr); if idx == 0 { return None; @@ -472,6 +562,16 @@ impl ValidPointerSet { self.arena_blocks.get(idx - 1) } + /// `(direct-mapped answer, binary-search answer)` as block bases, for the + /// block-index test. + #[cfg(test)] + pub(super) fn census_block_base_both_ways(&self, ptr: usize) -> (Option, Option) { + ( + self.census_block_at(ptr).map(|b| b.base), + self.census_block_at_by_search(ptr).map(|b| b.base), + ) + } + /// Exact arena membership: a census hit also records that the trace /// reached the hit's block (#10182; see `block_skip`'s module doc for why /// every mark passes through here). @@ -559,6 +659,36 @@ impl ValidPointerSet { } } +/// Sabotage switch for the block-index test: a lookup ignores the base inside +/// the window and answers with the block below it. Test builds only. +#[cfg(test)] +pub(crate) mod block_window_sabotage { + use std::cell::Cell; + + thread_local! { + static IGNORE_INSIDE: Cell = const { Cell::new(false) }; + } + + #[inline] + pub(crate) fn ignoring_inside() -> bool { + IGNORE_INSIDE.with(Cell::get) + } + + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(IGNORE_INSIDE.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + IGNORE_INSIDE.with(|s| s.set(self.0)); + } + } +} + /// Sabotage switch for the whole-block census test: the one-pass walk skips /// recording every start bit. Test builds only. #[cfg(test)] @@ -749,6 +879,7 @@ impl ValidPointerSetBuilder { return false; } self.set.block_census.flush_block(); + self.set.build_block_windows(); self.phase = ValidPointerSetBuildPhase::Done; return true; } From 48f96f1d4860afdcb0070241e64bfb5262c49d4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 23:54:11 +0200 Subject: [PATCH 18/40] gc: mark a pointer-free object without queueing it (#10182) mark_field_into_worklist queued every newly marked object, and the drain then dispatched each one only to find a leaf descriptor with no slot to visit. Strings are half the objects of a parsed JSON tree. A pointer-free object that is not a forwarding stub is now marked and not queued; a forwarded one is still queued so its hop is followed. --- .../perry-runtime/src/gc/tests/leaf_marks.rs | 80 +++++++++++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + crates/perry-runtime/src/gc/trace.rs | 44 ++++++++++ 3 files changed, 125 insertions(+) create mode 100644 crates/perry-runtime/src/gc/tests/leaf_marks.rs diff --git a/crates/perry-runtime/src/gc/tests/leaf_marks.rs b/crates/perry-runtime/src/gc/tests/leaf_marks.rs new file mode 100644 index 0000000000..c5827c6baa --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/leaf_marks.rs @@ -0,0 +1,80 @@ +//! #10182: the full mark does not queue a pointer-free object that is not a +//! forwarding stub; tracing it would do nothing. +//! +//! A rooted old array holds strings (leaves) and one string header forwarded to +//! another string, the shape old-page evacuation leaves behind. After a full, +//! every held string survives and so does the forwarding target, reachable only +//! through the stub's hop. The sabotaged twin skips queueing the forwarded leaf +//! too, and the target is swept. + +use super::super::*; +use super::support::*; +use crate::gc::trace::leaf_mark_sabotage; + +/// Returns `(held strings surviving, held strings, target survived)`. +fn collect(sabotaged: bool) -> (usize, usize, bool) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + unsafe { + let (holder, elements) = alloc_old_test_array(64); + let mut root = ptr_bits(holder as usize); + js_gc_register_global_root(&mut root as *mut u64 as i64); + let mut held = Vec::new(); + for i in 0..63usize { + let user = crate::arena::arena_alloc_gc_old(24 + i, 8, GC_TYPE_STRING) as usize; + *elements.add(i) = string_bits(user); + held.push(user); + // Dead neighbours, so a missed mark would really be reclaimed. + crate::arena::arena_alloc_gc_old(24, 8, GC_TYPE_STRING); + } + let stub = crate::arena::arena_alloc_gc_old(24, 8, GC_TYPE_STRING) as usize; + let target = crate::arena::arena_alloc_gc_old(32, 8, GC_TYPE_STRING) as usize; + crate::gc::set_forwarding_address( + header_from_user_ptr(stub as *const u8) as *mut GcHeader, + target as *mut u8, + ); + *elements.add(63) = string_bits(stub); + { + let _sabotage = sabotaged.then(leaf_mark_sabotage::Guard::arm); + let _ = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); + } + let survived = held + .iter() + .filter(|&&u| (*header_from_user_ptr(u as *const u8)).obj_type == GC_TYPE_STRING) + .count(); + let target_alive = + (*header_from_user_ptr(target as *const u8)).obj_type == GC_TYPE_STRING; + (survived, held.len(), target_alive) + } + }) + .join() + .expect("leaf-mark test thread must not panic") +} + +#[test] +fn unqueued_leaf_marks_keep_leaves_and_a_forwarded_leaf_still_hops() { + let (survived, held, target_alive) = collect(false); + assert_eq!(survived, held, "every held string must survive"); + assert!( + target_alive, + "the forwarding target must be reached through the stub" + ); +} + +#[test] +fn sabotaged_forwarded_leaf_loses_its_target() { + let (survived, held, target_alive) = collect(true); + assert_eq!( + survived, held, + "plain leaves are unaffected by the sabotage" + ); + assert!( + !target_alive, + "a forwarded leaf that is not queued never hops, and its target is swept" + ); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index b316078f04..16f913deae 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -49,6 +49,7 @@ mod layout_residue_histogram; mod layout_trace; mod lazy_intrinsic_towers; mod lazy_tape_side_alloc; +mod leaf_marks; mod mark_slot_hoists; mod oldgen; mod os_tag; diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 918d7b62a6..9a9bd729a3 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -1285,6 +1285,19 @@ pub(super) unsafe fn mark_field_into_worklist( return false; } (*header).gc_flags = flags | GC_FLAG_MARKED; + // #10182: tracing a pointer-free object that is not a forwarding stub does + // nothing — `trace_one_worklist_header` would only follow a FORWARDED hop, + // and a leaf descriptor visits no slot — so it is marked and not queued. + // Strings are half the objects of a JSON tree. + #[cfg(not(test))] + let forwarded = flags & GC_FLAG_FORWARDED != 0; + #[cfg(test)] + let forwarded = flags & GC_FLAG_FORWARDED != 0 && !leaf_mark_sabotage::ignoring_forwarding(); + if !forwarded + && gc_type_rewrite_descriptor_kind((*header).obj_type) == GcRewriteDescriptorKind::Leaf + { + return true; + } // Push directly onto the caller's worklist. No MARK_SEEDS push — // that's only needed for root-phase callers that don't own a // worklist (mark_mutable_root_slots, mark_registered_roots, @@ -1294,6 +1307,37 @@ pub(super) unsafe fn mark_field_into_worklist( true } +/// Sabotage switch for the leaf-mark test: a forwarded pointer-free object is +/// not queued either, so its forwarding hop is never followed. Test builds +/// only. +#[cfg(test)] +pub(crate) mod leaf_mark_sabotage { + use std::cell::Cell; + + thread_local! { + static IGNORE_FORWARDING: Cell = const { Cell::new(false) }; + } + + #[inline] + pub(crate) fn ignoring_forwarding() -> bool { + IGNORE_FORWARDING.with(Cell::get) + } + + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(IGNORE_FORWARDING.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + IGNORE_FORWARDING.with(|s| s.set(self.0)); + } + } +} + pub(super) fn try_mark_young_value_as_seed(value_bits: u64, valid_ptrs: &ValidPointerSet) -> bool { let ptr = decode_heap_addr(value_bits); try_mark_young_user_ptr_as_seed(ptr, valid_ptrs) From cd93f0adc61261455c9fde23cc550ba9e6bc5aad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 00:12:40 +0200 Subject: [PATCH 19/40] gc: prefetch ahead in the full mark's worklist drain (#10182) The copying minor's drain prefetches the header a few worklist entries ahead; the full mark's drain did not, and on a promoted 20 MB JSON tree each header it dequeues is a cold DRAM read. A prefetch has no architectural effect and cannot fault. --- crates/perry-runtime/src/gc/trace.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 9a9bd729a3..9820dda04b 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -1412,6 +1412,13 @@ pub(super) fn drain_trace_worklist_step( let mut remaining = budget; while remaining > 0 && *cursor < worklist.len() { let header = worklist[*cursor]; + // #10182: the drain visits headers in queue order and each one is a + // cold DRAM read on a heap larger than the cache (a 20 MB JSON tree); + // start the read of the entry a few places ahead, as the copying + // minor's drain already does. A prefetch cannot fault. + if let Some(&ahead) = worklist.get(*cursor + super::prefetch::PREFETCH_DISTANCE) { + super::prefetch::prefetch_read(ahead as usize); + } *cursor += 1; trace_one_worklist_header(header, valid_ptrs, worklist, minor_only); remaining -= 1; From 8288faee6037613353deb2af07813dbcf3d4d837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 00:14:15 +0200 Subject: [PATCH 20/40] gc: pace a full by the bytes promoted since the last full, at the promoting safepoint (#10182) Old-reclaim pacing cannot see a promoted tree that dies after its minor: every promotion credits the growth baseline (#7592/#7965). A document parse/scan loop lives in that blind spot, because each result's top-level array is born old and keeps its young contents reachable through remembered slots until a full proves it dead; on records_array_20m:parse every minor promotes two trees, one of them dead, and nothing ever collects them. A full is now due when the bytes promoted since the last full reach max(one base nursery, the old-gen live set that full verified << backoff). It is consulted only right after a nursery minor at a precise safepoint, so the full runs with precise roots and, after an in-place promotion, an empty young generation. It is not an arm of old_reclaim_pressure_due, whose credited baseline and the two tests #10204 broke are untouched. A cohort full that reclaims less than half its cohort doubles the bound (at most three times), so a heap whose promoted bytes stay live pays a logarithmic number of futile fulls; a productive one restores it. Diag: [gc-trigger] promoted_since_full= cohort_bound=, site safepoint_promoted_cohort, [gc-promoted-cohort] full cohort= bound= reclaimed= productive= backoff_shift=. --- crates/perry-runtime/src/gc/diag_sites.rs | 4 +- crates/perry-runtime/src/gc/mod.rs | 2 + crates/perry-runtime/src/gc/policy.rs | 39 ++++ .../perry-runtime/src/gc/promoted_cohort.rs | 165 ++++++++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/promoted_cohort.rs | 186 ++++++++++++++++++ 6 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 crates/perry-runtime/src/gc/promoted_cohort.rs create mode 100644 crates/perry-runtime/src/gc/tests/promoted_cohort.rs diff --git a/crates/perry-runtime/src/gc/diag_sites.rs b/crates/perry-runtime/src/gc/diag_sites.rs index 5d7776bfab..3b2f83bb79 100644 --- a/crates/perry-runtime/src/gc/diag_sites.rs +++ b/crates/perry-runtime/src/gc/diag_sites.rs @@ -59,7 +59,9 @@ pub(super) fn trigger_decision(site: &'static str, kind: &'static str) { from_space={from_space} nursery_cap={nursery_cap} old_in_use={old_in_use} old_free={old_free} \ old_reclaimable={old_reclaimable} external_side={external} old_baseline={old_baseline} \ old_band={old_band} old_threshold={old_threshold} old_pending={old_pending} retaining={retaining} \ - malloc={malloc} next_malloc={next_malloc}" + malloc={malloc} next_malloc={next_malloc} promoted_since_full={} cohort_bound={}", + promoted_cohort::promoted_since_full(), + promoted_cohort::bound_bytes() ); } diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 09f92ead01..2926edf988 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -236,6 +236,8 @@ mod native_stack_scan; /// mechanism is `arena/promote.rs`; this decides when to use it. mod promote_in_place; use promote_in_place::*; +/// #10182: full collections paced by the bytes promoted since the last full. +mod promoted_cohort; #[cfg(test)] pub(crate) use promote_in_place::{ clear_young_survival_for_tests, last_young_survival_permille, seed_young_survival_for_tests, diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 1c33d1e9d4..b957f2f304 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1993,6 +1993,9 @@ pub(super) fn credit_promoted_bytes_to_old_baseline(promoted_bytes: usize) { } GC_LAST_OLD_RECLAIM_IN_USE_BYTES .with(|bytes| bytes.set(bytes.get().saturating_add(promoted_bytes))); + // #10182: the credit hides these bytes from the growth band by design; the + // promoted-cohort bound is what still counts them. + super::promoted_cohort::note_promoted(promoted_bytes); } /// Feed a copying minor's measured young-survival ratio to arena-growth pacing. @@ -2070,6 +2073,8 @@ pub(super) fn finish_full_old_reclaim_baseline() { let old_in_use = old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|bytes| bytes.set(old_in_use)); + // #10182: this full verified everything old; the promoted cohort starts over. + super::promoted_cohort::note_full_finished(old_in_use); // Record the TOTAL post-full live set for major-GC pacing (young+old): the // full sweep is the only collection that frees forwarding stubs, so this is // the "clean" size the arena returns to and the base for the K× growth gate. @@ -3538,6 +3543,40 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { // the precise collection that replaced it actually ran (CLAUDE.md, four // ways a gate cannot fail — #4, the gate runs but its subject never did). super::record_safepoint_drain(super::SafepointDrainKind::NurseryMinor); + run_promoted_cohort_full_if_due(); + true +} + +/// #10182: run the promoted-cohort full (`gc::promoted_cohort`) if the nursery +/// minor this precise safepoint just ran brought the cohort to its bound. +/// +/// Same collection the OldReclaim safepoint arm runs — a synchronous full with +/// `SkipDisabled` roots — at the same kind of point, and right after a minor, +/// so an in-place promotion has left no young object for the remembered-set +/// rebuild to find. Returns whether a full ran. +pub(super) fn run_promoted_cohort_full_if_due() -> bool { + if !super::promoted_cohort::full_due() || GC_OLD_RECLAIM_IN_PROGRESS.with(Cell::get) { + return false; + } + let _reentry = OldReclaimReentryGuard::enter(); + let cohort = super::promoted_cohort::promoted_since_full(); + let bound = super::promoted_cohort::bound_bytes(); + let before = old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); + super::diag_sites::trigger_decision("safepoint", "PromotedCohort"); + super::diag_sites::set_full_site("safepoint_promoted_cohort"); + // No `force_full_scan`: roots are precise at this safepoint. + gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::OldGenBytes)) + .emit_after_current(); + let after = old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); + let reclaimed = before.saturating_sub(after); + let productive = super::promoted_cohort::record_full_yield(cohort, reclaimed); + if super::gc_diag_enabled() { + eprintln!( + "[gc-promoted-cohort] full cohort={cohort} bound={bound} reclaimed={reclaimed} \ + productive={productive} backoff_shift={}", + super::promoted_cohort::backoff_shift() + ); + } true } diff --git a/crates/perry-runtime/src/gc/promoted_cohort.rs b/crates/perry-runtime/src/gc/promoted_cohort.rs new file mode 100644 index 0000000000..b9d0e9568e --- /dev/null +++ b/crates/perry-runtime/src/gc/promoted_cohort.rs @@ -0,0 +1,165 @@ +//! #10182: a full collection paced by the bytes promoted into old-gen since the +//! last full — the *promoted cohort* — rather than by old-gen growth. +//! +//! # Why the growth band cannot see this garbage +//! +//! Old-reclaim pacing measures `old_in_use - baseline`, and every promotion +//! credits the baseline (`credit_promoted_bytes_to_old_baseline`, #7592/#7965): +//! bytes a minor just moved into old-gen are growth the pacing decision has +//! already seen, and withholding the credit degenerates the band into a +//! constant on every retaining workload. So a promoted tree that dies after its +//! minor is invisible to old-reclaim. A document parse/scan loop lives in +//! exactly that blind spot: each parse result's top-level array is born old, +//! its young contents stay reachable through that array's remembered slots +//! until a full proves the array dead, so every nursery minor promotes the +//! previous (dead) tree together with the current one. On +//! `records_array_20m:parse` that is two trees per minor, one of them dead, and +//! old-gen holding every tree ever parsed until something else forces a full. +//! +//! # The bound +//! +//! A full is due when the cohort reaches `max(floor, live << backoff)`, where +//! `live` is the old-gen occupancy the last full verified and `floor` is one +//! base nursery. Each full costs O(live), so one full per `live` promoted bytes +//! keeps collector work proportional to allocation (the #7592 argument, applied +//! to promotions instead of arena growth) while capping how many dead cohorts +//! old-gen can hold. +//! +//! # Why this is not an old-reclaim arm +//! +//! #10204 measured the bound as a disjunct of `old_reclaim_pressure_due`. That +//! reintroduced the futile full `test_old_reclaim_band_is_proportional_and_promotion_credits_baseline` +//! and `an_untraced_promotion_credits_the_old_reclaim_baseline` pin: on a heap +//! whose promoted bytes are live (`retain`), a full scheduled because promotion +//! moved them frees nothing, and an allocation-point arm fires it behind a +//! forced conservative scan. Here, instead: +//! +//! * the bound is consulted in exactly one place, at a precise safepoint +//! right after the nursery minor it applies to (`gc_safepoint_moving_minor`), +//! so the full runs with precise roots and, after an in-place promotion, an +//! empty young generation (its remembered-set rebuild is provably empty); +//! * `old_reclaim_pressure_due` and the credited baseline are untouched; +//! * a cohort full that reclaims less than half the cohort doubles the bound +//! (up to `BACKOFF_SHIFT_MAX`), so a retaining heap pays a logarithmic number +//! of futile fulls, each O(live), and a productive full restores it. + +use std::cell::Cell; + +/// A cohort full is productive when it reclaims at least this percentage of +/// the cohort it was scheduled for. +const PRODUCTIVE_PERCENT: usize = 50; +/// The bound doubles at most this many times on consecutive futile fulls. +const BACKOFF_SHIFT_MAX: u32 = 3; + +crate::perry_thread_local! { + /// Bytes promoted into old-gen since the last full collection. + static PROMOTED_SINCE_FULL: Cell = const { Cell::new(0) }; + /// Old-gen occupancy (reclaimable pressure plus external side bytes) the + /// last full collection left behind. + static OLD_LIVE_AT_LAST_FULL: Cell = const { Cell::new(0) }; + /// Consecutive futile cohort fulls, capped at `BACKOFF_SHIFT_MAX`. + static BACKOFF_SHIFT: Cell = const { Cell::new(0) }; + /// Cohort fulls run on this thread (live-subject counter). + static COHORT_FULLS: Cell = const { Cell::new(0) }; +} + +/// A minor moved `bytes` into old-gen. +pub(super) fn note_promoted(bytes: usize) { + PROMOTED_SINCE_FULL.with(|c| c.set(c.get().saturating_add(bytes))); +} + +/// A full collection finished and verified `old_live` bytes of old-gen. +pub(super) fn note_full_finished(old_live: usize) { + OLD_LIVE_AT_LAST_FULL.with(|c| c.set(old_live)); + PROMOTED_SINCE_FULL.with(|c| c.set(0)); +} + +pub(super) fn promoted_since_full() -> usize { + PROMOTED_SINCE_FULL.with(Cell::get) +} + +/// The cohort size at which a full becomes due. +pub(super) fn bound_bytes() -> usize { + bound_from( + super::policy::gc_scavenge_nursery_cap_bytes(), + OLD_LIVE_AT_LAST_FULL.with(Cell::get), + BACKOFF_SHIFT.with(Cell::get), + ) +} + +/// `max(floor, live << shift)`, saturating. +pub(super) fn bound_from(floor: usize, old_live: usize, shift: u32) -> usize { + floor.max( + old_live + .checked_shl(shift) + .filter(|v| v >> shift == old_live) + .unwrap_or(usize::MAX), + ) +} + +pub(super) fn full_due() -> bool { + promoted_since_full() >= bound_bytes() +} + +/// Price a finished cohort full: `reclaimed` old-gen bytes against the +/// `cohort` it was scheduled for. +pub(super) fn record_full_yield(cohort: usize, reclaimed: usize) -> bool { + let productive = reclaimed.saturating_mul(100) >= cohort.saturating_mul(PRODUCTIVE_PERCENT); + #[cfg(test)] + let productive = productive || sabotage::never_back_off(); + BACKOFF_SHIFT.with(|shift| { + if productive { + shift.set(0); + } else { + shift.set(shift.get().saturating_add(1).min(BACKOFF_SHIFT_MAX)); + } + }); + COHORT_FULLS.with(|c| c.set(c.get().saturating_add(1))); + productive +} + +pub(super) fn backoff_shift() -> u32 { + BACKOFF_SHIFT.with(Cell::get) +} + +#[cfg(test)] +pub(super) fn cohort_fulls() -> u64 { + COHORT_FULLS.with(Cell::get) +} + +/// Sabotage switch for the cohort tests: every cohort full counts as +/// productive, so the bound never backs off. Test builds only. +#[cfg(test)] +pub(super) mod sabotage { + use std::cell::Cell; + + thread_local! { + static NEVER_BACK_OFF: Cell = const { Cell::new(false) }; + } + + pub(super) fn never_back_off() -> bool { + NEVER_BACK_OFF.with(Cell::get) + } + + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(NEVER_BACK_OFF.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + NEVER_BACK_OFF.with(|s| s.set(self.0)); + } + } +} + +/// Seed the cohort state (tests only). +#[cfg(test)] +pub(super) fn seed_for_tests(promoted_since_full: usize, old_live: usize, shift: u32) { + PROMOTED_SINCE_FULL.with(|c| c.set(promoted_since_full)); + OLD_LIVE_AT_LAST_FULL.with(|c| c.set(old_live)); + BACKOFF_SHIFT.with(|c| c.set(shift)); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 16f913deae..6af8642972 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -54,6 +54,7 @@ mod mark_slot_hoists; mod oldgen; mod os_tag; mod promote_in_place; +mod promoted_cohort; mod proxy_registry; mod restore_coverage; mod retention_9628_9629; diff --git a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs new file mode 100644 index 0000000000..deb1984774 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs @@ -0,0 +1,186 @@ +//! #10182: the promoted-cohort full (`gc::promoted_cohort`). +//! +//! The arithmetic, the futility backoff that keeps it from charging a +//! retaining heap a full per cohort, the proof that it is not an old-reclaim +//! arm (the two baseline tests #10204 broke keep their meaning), and one real +//! collection: a promoting minor credits the cohort, and the cohort full that +//! follows it reclaims a promoted object that died. + +use super::super::policy::{ + credit_promoted_bytes_to_old_baseline, old_reclaim_pressure_due, + run_promoted_cohort_full_if_due, GC_LAST_OLD_RECLAIM_IN_USE_BYTES, GC_MAJOR_PACING_RETAINING, +}; +use super::super::promoted_cohort as cohort; +use super::super::*; +use super::support::*; + +const MB: usize = 1024 * 1024; + +#[test] +fn the_bound_is_one_nursery_or_the_verified_live_set_doubled_per_futile_full() { + assert_eq!(cohort::bound_from(16 * MB, 0, 0), 16 * MB); + assert_eq!(cohort::bound_from(16 * MB, 8 * MB, 0), 16 * MB); + assert_eq!(cohort::bound_from(16 * MB, 49 * MB, 0), 49 * MB); + assert_eq!(cohort::bound_from(16 * MB, 49 * MB, 2), 196 * MB); + assert_eq!( + cohort::bound_from(16 * MB, usize::MAX / 2, 3), + usize::MAX, + "the shifted live set saturates instead of wrapping" + ); +} + +#[test] +fn a_futile_cohort_full_doubles_the_bound_and_a_productive_one_restores_it() { + let _iso = GcTestIsolationGuard::new(); + cohort::seed_for_tests(0, 49 * MB, 0); + assert!( + !cohort::record_full_yield(58 * MB, 28 * MB), + "48% is futile" + ); + assert_eq!(cohort::backoff_shift(), 1); + for _ in 0..8 { + cohort::record_full_yield(58 * MB, 0); + } + assert_eq!(cohort::backoff_shift(), 3, "the backoff is capped"); + assert!( + cohort::record_full_yield(58 * MB, 29 * MB), + "50% is productive" + ); + assert_eq!(cohort::backoff_shift(), 0); + cohort::seed_for_tests(0, 0, 0); +} + +/// Replay `retain`'s measured untraced promotion schedule (the one +/// `an_untraced_promotion_credits_the_old_reclaim_baseline` uses) through the +/// cohort arm on a heap where every promoted byte stays live, so every cohort +/// full is futile. Returns how many cohort fulls the schedule paid for. +fn retain_schedule_cohort_fulls() -> usize { + const RETAIN_UNTRACED_PROMOTION_BYTES: [usize; 4] = + [18_742_816, 26_213_656, 35_650_552, 37_747_640]; + cohort::seed_for_tests(0, 0, 0); + let mut old_live = 0usize; + let mut fulls = 0usize; + for step in RETAIN_UNTRACED_PROMOTION_BYTES.iter().cycle().take(64) { + old_live += step; + cohort::note_promoted(*step); + if cohort::full_due() { + let promoted = cohort::promoted_since_full(); + cohort::note_full_finished(old_live); + cohort::record_full_yield(promoted, 0); + fulls += 1; + } + } + cohort::seed_for_tests(0, 0, 0); + fulls +} + +#[test] +fn a_retaining_promotion_schedule_pays_a_bounded_number_of_futile_cohort_fulls() { + let _iso = GcTestIsolationGuard::new(); + let fulls = retain_schedule_cohort_fulls(); + assert!( + fulls <= 4, + "64 promotions of live data (~1.9 GB) may cost at most a handful of futile \ + fulls, each O(live): {fulls}" + ); +} + +#[test] +fn sabotaged_backoff_charges_a_retaining_schedule_a_full_per_live_set() { + let _iso = GcTestIsolationGuard::new(); + let fulls = { + let _sabotage = cohort::sabotage::Guard::arm(); + retain_schedule_cohort_fulls() + }; + assert!( + fulls > 4, + "without the backoff the same schedule pays a full each time the cohort \ + reaches the live set: {fulls}" + ); +} + +/// The two baseline tests #10204 broke by adding the bound to +/// `old_reclaim_pressure_due` pin that promoted bytes alone never make old +/// reclaim due. The cohort is not consulted there, however large it is. +#[test] +fn the_cohort_never_makes_old_reclaim_due() { + let _iso = GcTestIsolationGuard::new(); + let previous_retaining = GC_MAJOR_PACING_RETAINING.with(std::cell::Cell::get); + let previous_baseline = GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(std::cell::Cell::get); + GC_MAJOR_PACING_RETAINING.with(|c| c.set(true)); + GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|c| c.set(4 * MB)); + cohort::seed_for_tests(0, 0, 0); + credit_promoted_bytes_to_old_baseline(270 * MB); + assert!( + cohort::full_due(), + "premise: the cohort is far past its bound" + ); + let baseline = GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(std::cell::Cell::get); + assert!(!old_reclaim_pressure_due(274 * MB, baseline)); + cohort::seed_for_tests(0, 0, 0); + GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|c| c.set(previous_baseline)); + GC_MAJOR_PACING_RETAINING.with(|c| c.set(previous_retaining)); +} + +/// A real untraced in-place promotion credits the cohort by exactly the bytes +/// it moved; one of the promoted leaves then dies, and the cohort full reclaims +/// it and keeps the other. Below the bound the same state runs no full. +fn promote_two_leaves_then_drop_one(cohort_due: bool) -> (bool, u8, u8, usize) { + let _guard = CopyingNurseryTestGuard::new(4); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _promote = super::super::InPlacePromotionTestGuard::untraced(); + cohort::seed_for_tests(0, 0, 0); + + let kept = young_leaf(); + let dropped = young_leaf(); + js_shadow_slot_set(0, string_bits(kept)); + js_shadow_slot_set(1, string_bits(dropped)); + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert!( + trace.copying_nursery.in_place_promotion && trace.copying_nursery.promoted_bytes > 0, + "premise: the minor promoted in place" + ); + assert_eq!( + cohort::promoted_since_full(), + trace.copying_nursery.promoted_bytes, + "the cohort is credited with exactly the promoted bytes" + ); + assert!(crate::arena::pointer_in_old_gen(dropped)); + js_shadow_slot_set(1, crate::value::TAG_UNDEFINED); + if cohort_due { + cohort::seed_for_tests(cohort::bound_bytes(), 0, 0); + } + let fulls_before = cohort::cohort_fulls(); + let ran = run_promoted_cohort_full_if_due(); + assert_eq!(cohort::cohort_fulls() - fulls_before, u64::from(ran)); + let type_of = |user: usize| unsafe { (*header_from_user_ptr(user as *const u8)).obj_type }; + let result = ( + ran, + type_of(kept), + type_of(dropped), + cohort::promoted_since_full(), + ); + js_shadow_slot_set(0, crate::value::TAG_UNDEFINED); + cohort::seed_for_tests(0, 0, 0); + result +} + +#[test] +fn a_cohort_full_after_a_promotion_reclaims_the_promoted_object_that_died() { + let (ran, kept, dropped, cohort_after) = promote_two_leaves_then_drop_one(true); + assert!(ran, "the cohort at its bound must run the full"); + assert_eq!(kept, GC_TYPE_STRING, "the rooted promoted leaf survives"); + assert_eq!(dropped, 0, "the promoted leaf that died is reclaimed"); + assert_eq!(cohort_after, 0, "the full restarts the cohort"); +} + +#[test] +fn below_the_bound_no_cohort_full_runs_and_the_dead_promoted_object_stays() { + let (ran, kept, dropped, _) = promote_two_leaves_then_drop_one(false); + assert!(!ran); + assert_eq!(kept, GC_TYPE_STRING); + assert_eq!( + dropped, GC_TYPE_STRING, + "only a full can reclaim a promoted object" + ); +} From 84bd96fbabcdc6ed0f9296bee7fe28c36b3c13e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 00:17:59 +0200 Subject: [PATCH 21/40] gc: a promoted-cohort full adopts the census of the blocks its minor just promoted (#10182) The cohort full runs at the same safepoint as the nursery minor whose promotion made it due, and that minor's untraced in-place promotion has just parsed every header of every block it promoted. The full's census parsed them again before the mutator ran a single instruction; on records_array_20m:parse those blocks are two of the three trees the census reads. When the minor's promotion can bring the cohort to its bound, the promotion walk now records for each block it parses whole the census's start bitmap and per-block facts, the flag facts computed by the census's own function after the promotion's own header writes. Only the cohort full started at the same safepoint may adopt a record, only for a block with the same address, bump offset and size, and every record is discarded when the safepoint returns. Test builds re-walk every adopted block with the census walk and assert the record agrees. --- crates/perry-runtime/src/arena/promote.rs | 11 + crates/perry-runtime/src/gc/policy.rs | 20 +- .../perry-runtime/src/gc/promoted_cohort.rs | 5 + .../src/gc/tests/adopt_census.rs | 141 +++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + crates/perry-runtime/src/gc/trace.rs | 66 ++++ .../src/gc/trace/adopt_census.rs | 362 ++++++++++++++++++ .../perry-runtime/src/gc/trace/block_skip.rs | 62 ++- 8 files changed, 659 insertions(+), 9 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/adopt_census.rs create mode 100644 crates/perry-runtime/src/gc/trace/adopt_census.rs diff --git a/crates/perry-runtime/src/arena/promote.rs b/crates/perry-runtime/src/arena/promote.rs index 7d5bdbd3a3..cd3d4065e8 100644 --- a/crates/perry-runtime/src/arena/promote.rs +++ b/crates/perry-runtime/src/arena/promote.rs @@ -465,6 +465,14 @@ fn stamp_and_index_block(block: &ArenaBlock, liveness: PromotionLiveness) -> (us let mut run_count = 0usize; let mut run_bytes = 0usize; let mut run_headers: Vec = Vec::new(); + // #10182: record the census facts of an untraced promotion's blocks for the + // promoted-cohort full that may follow at this safepoint (`adopt_census`). + let mut census = if describe { + crate::gc::AdoptableBlockBuilder::begin(block.data as usize, block.offset, block.size) + } else { + crate::gc::AdoptableBlockBuilder::begin(block.data as usize, 0, block.size) + }; + let mut stopped_early = false; let mut offset = 0usize; while offset < block.offset { @@ -478,6 +486,7 @@ fn stamp_and_index_block(block: &ArenaBlock, liveness: PromotionLiveness) -> (us if total < crate::gc::GC_HEADER_SIZE || total > block.size - aligned { // Same guard the arena walkers use: an implausible size means we // have run off the end of the initialised region. + stopped_early = true; break; } let obj_type = unsafe { (*header).obj_type }; @@ -489,6 +498,7 @@ fn stamp_and_index_block(block: &ArenaBlock, liveness: PromotionLiveness) -> (us // be looked at again. unsafe { crate::gc::stamp_header_promoted_in_place(header); + census.note(header, aligned); } objects += 1; @@ -552,6 +562,7 @@ fn stamp_and_index_block(block: &ArenaBlock, liveness: PromotionLiveness) -> (us run_bytes, ); } + census.finish(!stopped_early); debug_assert_eq!( offset, block.offset, "a promoted block did not parse to its own bump offset — its tail is \ diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index b957f2f304..f40148bd57 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -3528,8 +3528,21 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { _ => "ArenaBytes", }, ); + // #10182: when this minor's promotion can bring the promoted cohort to its + // bound, its promotion walk records the census facts of the blocks it + // promotes for the cohort full that would follow at this safepoint. + let record_census = matches!(kind, GcTriggerKind::ArenaBytes) + && super::promoted_cohort::promotion_may_reach_bound( + crate::arena::copying_from_space_in_use_bytes(), + ); + if record_census { + super::trace::adopt_census::begin_recording(); + } // No `force_full_scan`: roots are precise at this safepoint. let outcome = super::gc_collect_minor_with_trigger(GcTriggerSnapshot::capture(kind)); + if record_census { + super::trace::adopt_census::finish_recording(); + } match kind { GcTriggerKind::MallocCount => { gc_finish_malloc_trigger_collection(pre_malloc_count, pre_in_use, outcome); @@ -3544,6 +3557,7 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { // ways a gate cannot fail — #4, the gate runs but its subject never did). super::record_safepoint_drain(super::SafepointDrainKind::NurseryMinor); run_promoted_cohort_full_if_due(); + super::trace::adopt_census::discard(); true } @@ -3564,16 +3578,20 @@ pub(super) fn run_promoted_cohort_full_if_due() -> bool { let before = old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); super::diag_sites::trigger_decision("safepoint", "PromotedCohort"); super::diag_sites::set_full_site("safepoint_promoted_cohort"); + let adopted_before = super::trace::adopt_census::adopted_blocks(); + super::trace::adopt_census::begin_adopting(); // No `force_full_scan`: roots are precise at this safepoint. gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::OldGenBytes)) .emit_after_current(); + super::trace::adopt_census::discard(); + let adopted = super::trace::adopt_census::adopted_blocks() - adopted_before; let after = old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); let reclaimed = before.saturating_sub(after); let productive = super::promoted_cohort::record_full_yield(cohort, reclaimed); if super::gc_diag_enabled() { eprintln!( "[gc-promoted-cohort] full cohort={cohort} bound={bound} reclaimed={reclaimed} \ - productive={productive} backoff_shift={}", + productive={productive} adopted_census_blocks={adopted} backoff_shift={}", super::promoted_cohort::backoff_shift() ); } diff --git a/crates/perry-runtime/src/gc/promoted_cohort.rs b/crates/perry-runtime/src/gc/promoted_cohort.rs index b9d0e9568e..28bc6e18b7 100644 --- a/crates/perry-runtime/src/gc/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/promoted_cohort.rs @@ -97,6 +97,11 @@ pub(super) fn bound_from(floor: usize, old_live: usize, shift: u32) -> usize { ) } +/// Could a promotion of `young_bytes` bring the cohort to its bound? +pub(super) fn promotion_may_reach_bound(young_bytes: usize) -> bool { + promoted_since_full().saturating_add(young_bytes) >= bound_bytes() +} + pub(super) fn full_due() -> bool { promoted_since_full() >= bound_bytes() } diff --git a/crates/perry-runtime/src/gc/tests/adopt_census.rs b/crates/perry-runtime/src/gc/tests/adopt_census.rs new file mode 100644 index 0000000000..46342938b0 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/adopt_census.rs @@ -0,0 +1,141 @@ +//! #10182: a promoted-cohort full adopts the census record the in-place +//! promotion walk made of each block it promoted, instead of walking it again. +//! +//! A young population of about two megabytes — a rooted young array of young +//! arrays holding young strings, with dead strings between them — is promoted +//! whole by an untraced minor while recording is armed. Half the held strings +//! then die, and the cohort full runs. Every adopted block is re-walked by the +//! census in test builds (`adopt_census::verify_adopted_block`), the held +//! strings survive and the dropped ones are reclaimed. The sabotaged twin +//! records blocks without start bits: the census then cannot recognise any +//! object in them, and the rooted strings are swept. + +use super::super::policy::run_promoted_cohort_full_if_due; +use super::super::promoted_cohort as cohort; +use super::super::trace::adopt_census; +use super::super::*; +use super::support::*; + +struct Outcome { + adopted: u64, + kept_alive: usize, + kept: usize, + dropped_reclaimed: usize, + dropped: usize, +} + +fn promote_then_collect_cohort(sabotaged: bool) -> Outcome { + let _guard = CopyingNurseryTestGuard::new(4); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _promote = super::super::InPlacePromotionTestGuard::untraced(); + cohort::seed_for_tests(0, 0, 0); + + const INNER: usize = 64; + const OUTER: usize = 64; + let mut outer = crate::array::js_array_alloc(OUTER as u32); + let mut held: Vec<(usize, usize, usize)> = Vec::new(); + for o in 0..OUTER { + let mut inner = crate::array::js_array_alloc(INNER as u32); + for i in 0..INNER { + let leaf = young_leaf(); + inner = crate::array::js_array_push_jsvalue(inner, string_bits(leaf)); + held.push((o, i, leaf)); + for _ in 0..8 { + young_leaf(); + } + } + outer = crate::array::js_array_push_jsvalue(outer, ptr_bits(inner as usize)); + } + js_shadow_slot_set(0, ptr_bits(outer as usize)); + assert!( + crate::arena::pointer_in_nursery(held[0].2), + "premise: young population" + ); + + adopt_census::begin_recording(); + let trace = { + let _sabotage = sabotaged.then(adopt_census::sabotage::Guard::arm); + collect_minor_trace(GcTriggerKind::Direct) + }; + adopt_census::finish_recording(); + assert!( + trace.copying_nursery.in_place_promotion, + "premise: the minor promoted in place" + ); + assert!(crate::arena::pointer_in_old_gen(held[0].2)); + + // Drop every other held string. + let mut kept = Vec::new(); + let mut dropped = Vec::new(); + for &(o, i, leaf) in &held { + if (o + i) % 2 == 0 { + kept.push(leaf); + } else { + let inner = crate::array::js_array_get_jsvalue(outer, o as u32); + crate::array::js_array_set_jsvalue( + ((inner & POINTER_MASK) as usize) as *mut crate::array::ArrayHeader, + i as u32, + crate::value::TAG_UNDEFINED, + ); + dropped.push(leaf); + } + } + + cohort::seed_for_tests(cohort::bound_bytes(), 0, 0); + let adopted_before = adopt_census::adopted_blocks(); + let ran = { + let _sabotage = sabotaged.then(adopt_census::sabotage::Guard::arm); + run_promoted_cohort_full_if_due() + }; + assert!(ran, "premise: the cohort full ran"); + let adopted = adopt_census::adopted_blocks() - adopted_before; + let alive = |user: usize| { + crate::arena::pointer_in_old_gen(user) + && unsafe { (*header_from_user_ptr(user as *const u8)).obj_type } == GC_TYPE_STRING + }; + let outcome = Outcome { + adopted, + kept_alive: kept.iter().filter(|&&u| alive(u)).count(), + kept: kept.len(), + dropped_reclaimed: dropped.iter().filter(|&&u| !alive(u)).count(), + dropped: dropped.len(), + }; + js_shadow_slot_set(0, crate::value::TAG_UNDEFINED); + cohort::seed_for_tests(0, 0, 0); + adopt_census::discard(); + outcome +} + +#[test] +fn a_cohort_full_adopts_the_promotion_census_and_collects_exactly() { + let outcome = promote_then_collect_cohort(false); + assert!( + outcome.adopted >= 2, + "the full must adopt the promoted blocks' records: {}", + outcome.adopted + ); + assert_eq!( + outcome.kept_alive, outcome.kept, + "every held string survives" + ); + assert_eq!( + outcome.dropped_reclaimed, outcome.dropped, + "every dropped string is reclaimed" + ); +} + +#[test] +fn sabotaged_promotion_census_loses_rooted_objects() { + let outcome = promote_then_collect_cohort(true); + assert!( + outcome.adopted >= 2, + "premise: the sabotaged records were adopted" + ); + assert!( + outcome.kept_alive < outcome.kept, + "records without start bits hide every object in their blocks from the \ + mark, so rooted strings are swept: {} of {} survived", + outcome.kept_alive, + outcome.kept + ); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 6af8642972..40c0dc9dff 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -1,3 +1,4 @@ +mod adopt_census; mod alloc; mod arena_right_size; mod arguments_objects; diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 9820dda04b..6346ae8045 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -4,6 +4,10 @@ use super::*; pub(super) mod block_skip; pub(super) use block_skip::BlockCensus; +#[path = "trace/adopt_census.rs"] +pub(crate) mod adopt_census; +pub(crate) use adopt_census::AdoptableBlockBuilder; + crate::perry_thread_local! { /// Set by test-only helpers that wipe page metadata for isolation /// (`old_arena_page_index_clear_for_tests`): real objects become @@ -305,6 +309,49 @@ impl ValidPointerSet { }); } + /// Open census block `block_idx` with start-bitmap storage built by someone + /// else (`adopt_census`): `chunks` hold `words` words in the census layout + /// for a block whose walked extent is `offset`. + pub(super) fn begin_arena_block_with_chunks( + &mut self, + block_idx: u32, + data: usize, + offset: usize, + words: usize, + mut chunks: Vec>, + ) { + debug_assert!(!self.classifier_mode && offset <= CENSUS_BITMAP_MAX_EXTENT); + if let Some(previous) = self.arena_blocks.last() { + assert!( + previous.base.saturating_add(previous.extent) <= data, + "census blocks must arrive in ascending, non-overlapping address order: \ + {:#x}+{:#x} then {data:#x}", + previous.base, + previous.extent + ); + } + assert_eq!( + words, + offset.div_ceil(1 << CENSUS_START_ALIGN_SHIFT).div_ceil(64), + "adopted bitmap must cover exactly the block's walked extent" + ); + let mut pointers = [std::ptr::null_mut(); CENSUS_BITMAP_MAX_CHUNKS]; + for (index, chunk) in chunks.iter_mut().enumerate().take(CENSUS_BITMAP_MAX_CHUNKS) { + pointers[index] = chunk.as_mut_ptr(); + } + self.start_bitmap_chunks.extend(chunks); + self.arena_block_bases.push(data); + self.arena_blocks.push(CensusStartBlock { + base: data, + extent: offset, + block_idx, + chunks: pointers, + first: 0, + len: words, + sorted: false, + }); + } + /// Record a censused arena start (user pointer) in the block opened by the /// last `begin_arena_block`. Starts arrive in ascending address order — /// `ValidPointerSetBuilder` feeds them from `ArenaObjectCursor` in address @@ -995,6 +1042,25 @@ impl ValidPointerSetBuilder { data: usize, offset: usize, size: usize, + ) { + // #10182: a block the in-place promotion walk of this very safepoint + // recorded is adopted instead of walked (see `adopt_census`). + if self.adopt_promoted_block(block_idx, data, offset, size) { + return; + } + self.walk_census_block(block_idx, data, offset, size); + } + + /// The census walk of one whole block (see [`Self::census_whole_block`]). + /// + /// # Safety + /// As `census_whole_block`. + pub(super) unsafe fn walk_census_block( + &mut self, + block_idx: usize, + data: usize, + offset: usize, + size: usize, ) { let mut cursor = 0usize; let mut begun = false; diff --git a/crates/perry-runtime/src/gc/trace/adopt_census.rs b/crates/perry-runtime/src/gc/trace/adopt_census.rs new file mode 100644 index 0000000000..292ecf1f47 --- /dev/null +++ b/crates/perry-runtime/src/gc/trace/adopt_census.rs @@ -0,0 +1,362 @@ +//! Census facts of promoted blocks, recorded by the in-place promotion walk and +//! adopted by the promoted-cohort full that follows it (#10182). +//! +//! # Why +//! +//! A promoted-cohort full (`gc::promoted_cohort`) runs at the same precise +//! safepoint as the nursery minor whose promotion made it due. That minor's +//! untraced in-place promotion has just parsed every header of every block it +//! promoted (`arena::promote::stamp_and_index_block`), and the full's census +//! would parse them all again before the mutator has run a single instruction. +//! On `records_array_20m:parse` those blocks are two of the three JSON trees +//! the census reads. So the promotion walk records, per block, exactly what the +//! census records — the start bitmap and the per-block facts — and the census +//! adopts the record instead of walking the block. +//! +//! # Why the record equals the walk +//! +//! * **Same headers.** The promotion walk and the census walk use the same +//! alignment and hop; a record is kept only when the promotion walk parsed its +//! block to the bump offset without stopping on an implausible size, and a +//! parse that did not stop there cannot stop in the census walk either (the +//! census's size guard is weaker). +//! * **Same facts, read after the promotion's own header writes.** Each header +//! is noted after `stamp_header_promoted_in_place`, and the flag facts are +//! computed by the census's own function (`census_header_flag_facts`). +//! * **Nothing writes those headers in between.** Recording is armed only for +//! the nursery minor of one safepoint; the record can be adopted only by the +//! cohort full started at that same safepoint, before any mutator code runs, +//! and it is discarded when the safepoint returns. What the rest of the minor +//! does to promoted headers is clear `GC_FLAG_MARKED`, which can only turn a +//! recorded pre-marked fact into a conservative one. +//! * **Same block.** A record is adopted only for a block whose data address, +//! bump offset and size are the ones the census snapshotted, and only for an +//! old-generation block. +//! +//! Test builds re-walk every adopted block with the census walk and assert the +//! two agree, so every test that reaches adoption checks the argument. + +use super::*; +use std::cell::{Cell, RefCell}; + +/// One promoted block's census record. +pub(crate) struct AdoptableBlock { + extent: usize, + size: usize, + words: usize, + chunks: Vec>, + objects: usize, + bytes: u64, + first_start: usize, + last_start: usize, + /// Bit `t` of word `t >> 6` is set when an object of `obj_type == t` was + /// recorded; the census applies its own per-type obligations at adoption. + types: [u64; 4], + flag_obligation: bool, + premarked: bool, + non_walkable: bool, +} + +enum State { + Off, + Recording(crate::fast_hash::PtrHashMap), + Ready(crate::fast_hash::PtrHashMap), + Adopting(crate::fast_hash::PtrHashMap), +} + +crate::perry_thread_local! { + static STATE: RefCell = const { RefCell::new(State::Off) }; + /// Blocks the census adopted instead of walking (live-subject counter). + static ADOPTED_BLOCKS: Cell = const { Cell::new(0) }; +} + +/// Record the census facts of the blocks the next promotion walk promotes. +pub(crate) fn begin_recording() { + STATE.with(|s| *s.borrow_mut() = State::Recording(crate::fast_hash::new_ptr_hash_map())); +} + +fn recording() -> bool { + STATE.with(|s| matches!(*s.borrow(), State::Recording(_))) +} + +/// The minor is over: keep what it recorded for a full at this safepoint. +pub(crate) fn finish_recording() { + STATE.with(|s| { + let mut state = s.borrow_mut(); + *state = match std::mem::replace(&mut *state, State::Off) { + State::Recording(map) if !map.is_empty() => State::Ready(map), + _ => State::Off, + }; + }); +} + +/// The full about to start may adopt the records. +pub(crate) fn begin_adopting() { + STATE.with(|s| { + let mut state = s.borrow_mut(); + *state = match std::mem::replace(&mut *state, State::Off) { + State::Ready(map) => State::Adopting(map), + _ => State::Off, + }; + }); +} + +/// Drop every record: the safepoint is returning to the mutator. +pub(crate) fn discard() { + STATE.with(|s| *s.borrow_mut() = State::Off); +} + +/// Take the record of the block at `data`, if the census may adopt one. +fn take(data: usize) -> Option { + STATE.with(|s| match &mut *s.borrow_mut() { + State::Adopting(map) => map.remove(&data), + _ => None, + }) +} + +pub(crate) fn adopted_blocks() -> u64 { + ADOPTED_BLOCKS.with(Cell::get) +} + +/// Builds one block's record during the promotion walk. Inert when recording +/// is not armed or the block is too large for a start bitmap. +pub(crate) struct AdoptableBlockBuilder { + data: usize, + block: Option, +} + +impl AdoptableBlockBuilder { + pub(crate) fn begin(data: usize, offset: usize, size: usize) -> Self { + if offset == 0 || offset > CENSUS_BITMAP_MAX_EXTENT || !recording() { + return Self { data, block: None }; + } + let words = offset.div_ceil(1 << CENSUS_START_ALIGN_SHIFT).div_ceil(64); + let mut chunks = Vec::with_capacity(CENSUS_BITMAP_MAX_CHUNKS); + let mut start = 0; + while start < words { + chunks.push(vec![0u64; (words - start).min(CENSUS_BITMAP_CHUNK_WORDS)]); + start += CENSUS_BITMAP_CHUNK_WORDS; + } + Self { + data, + block: Some(AdoptableBlock { + extent: offset, + size, + words, + chunks, + objects: 0, + bytes: 0, + first_start: 0, + last_start: 0, + types: [0; 4], + flag_obligation: false, + premarked: false, + non_walkable: false, + }), + } + } + + /// Note the header at `aligned` bytes into the block, as the census would + /// see it now. + /// + /// # Safety + /// `header` is the header at `data + aligned`, below the block's offset. + #[inline] + pub(crate) unsafe fn note(&mut self, header: *const GcHeader, aligned: usize) { + let Some(block) = self.block.as_mut() else { + return; + }; + let obj_type = (*header).obj_type; + if !gc_type_is_arena_walkable(obj_type) { + block.non_walkable = true; + return; + } + let (flag_obligation, premarked) = block_skip::census_header_flag_facts(header); + block.flag_obligation |= flag_obligation; + block.premarked |= premarked; + block.types[(obj_type >> 6) as usize] |= 1u64 << (obj_type & 63); + let user_ptr = self.data + aligned + GC_HEADER_SIZE; + if block.objects == 0 { + block.first_start = user_ptr; + } + block.last_start = user_ptr; + block.objects += 1; + block.bytes += (*header).size as u64; + let bit = aligned >> CENSUS_START_ALIGN_SHIFT; + let word = bit >> 6; + block.chunks[word >> CENSUS_BITMAP_CHUNK_WORD_SHIFT] + [word & (CENSUS_BITMAP_CHUNK_WORDS - 1)] |= 1u64 << (bit & 63); + } + + /// Keep the record when the walk parsed the whole block. + pub(crate) fn finish(self, parsed_whole_block: bool) { + let Some(mut block) = self.block else { + return; + }; + if !parsed_whole_block { + return; + } + #[cfg(test)] + if sabotage::dropping_starts() { + for chunk in &mut block.chunks { + chunk.fill(0); + } + } + #[cfg(not(test))] + let _ = &mut block; + let data = self.data; + STATE.with(|s| { + if let State::Recording(map) = &mut *s.borrow_mut() { + map.insert(data, block); + } + }); + } +} + +impl ValidPointerSetBuilder { + /// Adopt the promotion walk's record of this block instead of walking it, + /// when one exists for exactly this block. Returns whether it did. + /// + /// # Safety + /// As `census_whole_block`. + pub(super) unsafe fn adopt_promoted_block( + &mut self, + block_idx: usize, + data: usize, + offset: usize, + size: usize, + ) -> bool { + if !self.census_armed || self.set.classifier_mode { + return false; + } + let Some(block) = take(data) else { + return false; + }; + if block.extent != offset + || block.size != size + || crate::arena::pointer_in_nursery(data + GC_HEADER_SIZE) + { + return false; + } + ADOPTED_BLOCKS.with(|c| c.set(c.get().saturating_add(1))); + if block.objects == 0 { + // The census opens no block for one without a walkable object. + return true; + } + #[cfg(test)] + let recorded = ( + block.chunks.clone(), + block.objects, + block.bytes, + block.first_start, + block.last_start, + block.non_walkable, + ); + self.census_block_idx = block_idx; + self.set.begin_arena_block_with_chunks( + u32::try_from(block_idx).unwrap_or(u32::MAX), + data, + offset, + block.words, + block.chunks, + ); + self.set.block_census.begin_block(block_idx, data, offset); + self.set.block_census.note_whole_block_walk(); + self.set.block_census.adopt_block_facts( + block.objects as u64, + block.bytes, + &block.types, + block.flag_obligation, + block.premarked, + block.non_walkable, + ); + self.set.arena_count += block.objects; + self.set.record_pointer_range(block.first_start); + self.set.record_pointer_range(block.last_start); + #[cfg(test)] + if !sabotage::dropping_starts() { + verify_adopted_block(block_idx, data, offset, size, &self.set, recorded); + } + true + } +} + +/// Test builds: walk the adopted block with the census walk into a scratch set +/// and assert the adopted record says the same. +#[cfg(test)] +unsafe fn verify_adopted_block( + block_idx: usize, + data: usize, + offset: usize, + size: usize, + adopted: &ValidPointerSet, + recorded: (Vec>, usize, u64, usize, usize, bool), +) { + let mut scratch = ValidPointerSetBuilder::new(); + scratch.walk_census_block(block_idx, data, offset, size); + scratch.set.block_census.flush_block(); + let (chunks, objects, bytes, first, last, non_walkable) = recorded; + let walked_block = scratch + .set + .arena_blocks + .last() + .expect("the walk opened the block"); + assert_eq!(walked_block.base, data); + assert_eq!(scratch.set.arena_count, objects, "adopted object count"); + assert_eq!( + scratch.set.start_bitmap_chunks, chunks, + "adopted start bitmap of block {data:#x}" + ); + assert_eq!( + (scratch.set.range_min, scratch.set.range_max), + (first, last) + ); + let walked = scratch + .set + .block_census + .block(block_idx) + .expect("walked facts"); + assert_eq!((walked.objects, walked.bytes), (objects as u64, bytes)); + assert_eq!( + walked.non_walkable, non_walkable, + "adopted non-walkable fact" + ); + let adopted_facts = adopted.block_census.current_facts_for_tests(); + assert!( + adopted_facts.obligation || !walked.obligation, + "an adopted record may only be more conservative than the walk (obligation)" + ); + assert!( + adopted_facts.premarked || !walked.premarked, + "an adopted record may only be more conservative than the walk (pre-marked)" + ); +} + +/// Sabotage switch for the adoption tests: records keep no start bits, and the +/// test-build verification stands down. Test builds only. +#[cfg(test)] +pub(crate) mod sabotage { + use std::cell::Cell; + + thread_local! { + static DROP_STARTS: Cell = const { Cell::new(false) }; + } + + pub(crate) fn dropping_starts() -> bool { + DROP_STARTS.with(Cell::get) + } + + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(DROP_STARTS.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + DROP_STARTS.with(|s| s.set(self.0)); + } + } +} diff --git a/crates/perry-runtime/src/gc/trace/block_skip.rs b/crates/perry-runtime/src/gc/trace/block_skip.rs index aab346fe24..46a11c81a4 100644 --- a/crates/perry-runtime/src/gc/trace/block_skip.rs +++ b/crates/perry-runtime/src/gc/trace/block_skip.rs @@ -175,23 +175,48 @@ impl BlockCensus { /// `header` must be a walkable arena header inside the current block. #[inline(always)] pub(crate) unsafe fn note_header(&mut self, header: *const GcHeader) { - let flags = (*header).gc_flags; let obj_type = (*header).obj_type; let size = (*header).size as u64; - let exceptional_flags = (flags ^ GC_FLAG_ARENA) - & (GC_FLAG_ARENA | GC_FLAG_MARKED | GC_FLAG_PINNED | GC_FLAG_FORWARDED) - != 0; - let premarked = flags & (GC_FLAG_MARKED | GC_FLAG_PINNED) != 0; - let raw_f64_array = obj_type == GC_TYPE_ARRAY - && (*header)._reserved & (GC_ARRAY_RAW_F64_LAYOUT | GC_ARRAY_RAW_F64_HOLES) != 0; + let (flag_obligation, premarked) = census_header_flag_facts(header); let type_obligation = self.obligation_by_type[obj_type as usize]; let block = &mut self.current; block.objects += 1; block.bytes += size; - block.obligation |= exceptional_flags | type_obligation | raw_f64_array; + block.obligation |= flag_obligation | type_obligation; block.premarked |= premarked; } + /// Set the current block's facts from a record another walk made of it + /// (`adopt_census`), applying this census's per-type obligations to the + /// recorded object types. + pub(crate) fn adopt_block_facts( + &mut self, + objects: u64, + bytes: u64, + types: &[u64; 4], + flag_obligation: bool, + premarked: bool, + non_walkable: bool, + ) { + if !self.armed { + return; + } + let type_obligation = (0..256usize) + .any(|t| types[t >> 6] & (1u64 << (t & 63)) != 0 && self.obligation_by_type[t]); + let block = &mut self.current; + block.objects = objects; + block.bytes = bytes; + block.obligation = flag_obligation || type_obligation; + block.premarked = premarked; + block.non_walkable = non_walkable; + } + + /// The facts of the block currently being censused (tests only). + #[cfg(test)] + pub(crate) fn current_facts_for_tests(&self) -> CensusBlock { + self.current + } + /// Fold the current block into the per-index table. Called at every block /// change and once when the census finishes. pub(crate) fn flush_block(&mut self) { @@ -351,6 +376,27 @@ pub(crate) mod sabotage { } } +/// The per-object census facts that depend on a header's flags rather than its +/// type: `(flag obligation, pre-marked)`. The flag obligation covers a pinned, +/// forwarded or already-marked header, one without `GC_FLAG_ARENA`, and an array +/// whose raw-f64 layout bits would fire a typed-feedback invalidation. +/// +/// # Safety +/// `header` is a readable arena header. +#[inline(always)] +pub(crate) unsafe fn census_header_flag_facts(header: *const GcHeader) -> (bool, bool) { + let flags = (*header).gc_flags; + let exceptional_flags = (flags ^ GC_FLAG_ARENA) + & (GC_FLAG_ARENA | GC_FLAG_MARKED | GC_FLAG_PINNED | GC_FLAG_FORWARDED) + != 0; + let raw_f64_array = (*header).obj_type == GC_TYPE_ARRAY + && (*header)._reserved & (GC_ARRAY_RAW_F64_LAYOUT | GC_ARRAY_RAW_F64_HOLES) != 0; + ( + exceptional_flags | raw_f64_array, + flags & (GC_FLAG_MARKED | GC_FLAG_PINNED) != 0, + ) +} + /// Does a dead object of `obj_type` need `reclaim_dead_object`'s per-object /// work beyond what the full trace's dead-owner fan-out and the block reset /// already do? From 4140ad173b2015e52c249f770832a9da52f1ba76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 00:32:47 +0200 Subject: [PATCH 22/40] gc: prefetch a range descriptor's children before marking them; inline the sweep's page tally (#10182) A range descriptor (an array's elements, an all-pointer field range) is now walked twice: the first pass starts the header read of every pointer-tagged slot's target, the second marks. account_old_object is inlined into the whole-block sweep's live path. --- crates/perry-runtime/src/gc/oldgen/sweep_objects.rs | 2 +- crates/perry-runtime/src/gc/trace.rs | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs index 64b6e2a8f8..23cdc39108 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs @@ -382,7 +382,7 @@ impl ArenaSweepObjectsState { /// Account one swept old object on its page(s), batching single-page /// objects per page. - #[inline] + #[inline(always)] fn account_old_object( &mut self, header: *mut GcHeader, diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 6346ae8045..669dde6f88 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -1770,6 +1770,19 @@ pub(super) unsafe fn trace_heap_rewrite_slots( } GcMutableSlotDescriptor::Slot(slot) => visit_slot(slot.slot, slot.layout_kind), GcMutableSlotDescriptor::Range { range, layout_kind } => { + // Start the header reads of the range's pointer children + // before marking any of them: each is a cold DRAM read the + // mark would otherwise take one at a time. A prefetch cannot + // fault, so the candidate need not be proven a pointer yet. + for i in 0..range.slot_count() { + let bits = *range.slot(i); + let tag = bits & TAG_MASK; + if tag == POINTER_TAG || tag == STRING_TAG { + super::prefetch::prefetch_read( + ((bits & POINTER_MASK) as usize).wrapping_sub(GC_HEADER_SIZE), + ); + } + } for i in 0..range.slot_count() { visit_slot(range.slot(i), layout_kind); } From 3eb2a43e3f3822761de6402881b521c139085623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 00:44:19 +0200 Subject: [PATCH 23/40] gc: census whole-block test plants enough objects for several bitmap blocks --- crates/perry-runtime/src/gc/tests/census_whole_block.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/gc/tests/census_whole_block.rs b/crates/perry-runtime/src/gc/tests/census_whole_block.rs index a8b7e9d018..c8c7766968 100644 --- a/crates/perry-runtime/src/gc/tests/census_whole_block.rs +++ b/crates/perry-runtime/src/gc/tests/census_whole_block.rs @@ -33,7 +33,7 @@ struct Population { } unsafe fn plant() -> Population { - for i in 0..6000usize { + for i in 0..40_000usize { let payload = [0usize, 8, 13, 40, 200][i % 5]; crate::arena::arena_alloc_gc_old(payload, 8, GC_TYPE_STRING); } From d7f5fd57c036bbe3d7622387cdcd8b7ae9b508e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 00:45:23 +0200 Subject: [PATCH 24/40] gc: classify the pacing-full counters and re-pin the census window (#10182) --- scripts/gc_runtime_root_holders.json | 44 +++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 7a38ba7164..f49a7563a6 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -313,7 +313,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -329,9 +329,9 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "5c151725460ffb92a55a6bee781123ef5159263b4ce5958d16570f78216e0d67", - "crates/perry-runtime/src/gc/cycle.rs": "b65e82014de18a1746fe9563f0c5bcef202f1a66e7497d51b4556b04c913db6e", - "crates/perry-runtime/src/gc/mod.rs": "9dbdde7594af06d08f45f82e426b220e586429da985e38f1e11e7f01e98a2527", - "crates/perry-runtime/src/gc/policy.rs": "abd08472fe71002a55a32f7a17021ab2c029d34ff2b26eb78340d1983ae939c4", + "crates/perry-runtime/src/gc/cycle.rs": "42c1654ab4ff7886da98ac36589a44080e0e10132460d245643cb14309d0ac80", + "crates/perry-runtime/src/gc/mod.rs": "9fedd2790f48154aaeceefb4805d3fbaa2fdf3c407529b326425fde86c2bf9a5", + "crates/perry-runtime/src/gc/policy.rs": "d588a0a3ffe21f8523d419fdced0942ca36b63c4d6c38faac93db1042304f8a9", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } } @@ -420,6 +420,30 @@ "verdict": "not_a_gc_pointer", "why": "#10169: a `Cell` scheduling flag — set when a document-sized JSON result was born old under young pressure, consumed by the next `gc_budgeted_due_trigger` decision. Holds no address of any kind." }, + { + "file": "crates/perry-runtime/src/gc/promoted_cohort.rs", + "name": "BACKOFF_SHIFT", + "verdict": "not_a_gc_pointer", + "why": "#10182: a `Cell` count of consecutive futile promoted-cohort fulls (capped at 3), the shift applied to the cohort bound. Holds no address." + }, + { + "file": "crates/perry-runtime/src/gc/promoted_cohort.rs", + "name": "COHORT_FULLS", + "verdict": "not_a_gc_pointer", + "why": "#10182 live-subject counter: a `Cell` count of promoted-cohort fulls this thread ran. A tally, never an address." + }, + { + "file": "crates/perry-runtime/src/gc/promoted_cohort.rs", + "name": "OLD_LIVE_AT_LAST_FULL", + "verdict": "not_a_gc_pointer", + "why": "#10182: a `Cell` byte count, the old-gen occupancy the last full collection verified, which the promoted-cohort bound is proportional to. Holds no address." + }, + { + "file": "crates/perry-runtime/src/gc/promoted_cohort.rs", + "name": "PROMOTED_SINCE_FULL", + "verdict": "not_a_gc_pointer", + "why": "#10182: a `Cell` byte count, the bytes promoted into old-gen since the last full collection, read by the promoted-cohort bound. Holds no address." + }, { "file": "crates/perry-runtime/src/gc/survival_diag.rs", "name": "MINOR_SEQ", @@ -438,6 +462,12 @@ "verdict": "not_a_gc_pointer", "why": "#9717: monotonic count of array-growth forwarding stubs a budgeted full cycle admitted through `classifier_valid_object_start`, reported as `forwarded_stub_recoveries=` on the PERRY_GC_DIAG `[gc-incremental]` line. A `Cell` holding a tally, never an address — the stubs it counts are reached through the worklist, not retained here. Nothing for the collector." }, + { + "file": "crates/perry-runtime/src/gc/trace/adopt_census.rs", + "name": "ADOPTED_BLOCKS", + "verdict": "not_a_gc_pointer", + "why": "#10182 live-subject counter: a `Cell` count of arena blocks whose census record the census adopted from the in-place promotion walk instead of walking them. A tally, never an address." + }, { "file": "crates/perry-runtime/src/gc/trace/block_skip.rs", "name": "BLOCK_SKIP_RECLAIMED_BLOCKS", @@ -456,6 +486,12 @@ "verdict": "not_a_gc_pointer", "why": "#10182 live-subject counter: a `Cell` count of objects in blocks reclaimed without a visit. An object count, never an address." }, + { + "file": "crates/perry-runtime/src/gc/trace/block_skip.rs", + "name": "HOLE_REBUILD_BLOCKS_SKIPPED", + "verdict": "not_a_gc_pointer", + "why": "#10182 live-subject counter: a `Cell` count of live old blocks a full sweep's hole-list rebuild did not parse because the census proved them hole-free. A tally, never an address." + }, { "file": "crates/perry-runtime/src/gc/trace/block_skip.rs", "name": "LAST_SKIPPED_BASES", From e38e3fc8e9d40c3f6a72e69af0c3c11ddf055ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 00:47:47 +0200 Subject: [PATCH 25/40] docs: promoted-cohort fulls and the one-pass full (#10182) --- crates/perry-runtime/src/gc/layout.rs | 15 +++------ docs/src/internals/garbage-collector.md | 41 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/crates/perry-runtime/src/gc/layout.rs b/crates/perry-runtime/src/gc/layout.rs index 97abde37a6..f5fbd30a8b 100644 --- a/crates/perry-runtime/src/gc/layout.rs +++ b/crates/perry-runtime/src/gc/layout.rs @@ -1878,19 +1878,12 @@ impl GcMutableSlot { Self { slot, layout_kind } } - /// Is the slot's own address outside the old generation? - /// - /// #10182: classified when asked, not when the slot is enumerated. The one - /// reader (the copying minor's `scan_object_fields`) asks immediately, so - /// the answer is the same; the full mark, which enumerates every traced - /// slot through this type and never asks, stopped paying a page-generation - /// lookup per slot. + /// Is the slot's address outside old-gen? #10182: classified on demand (its + /// one reader asks at once), so the full mark no longer classifies per slot. #[inline] pub(super) fn external(self) -> bool { - !matches!( - crate::arena::classify_heap_generation(self.slot as usize), - crate::arena::HeapGeneration::Old - ) + let generation = crate::arena::classify_heap_generation(self.slot as usize); + !matches!(generation, crate::arena::HeapGeneration::Old) } #[inline] diff --git a/docs/src/internals/garbage-collector.md b/docs/src/internals/garbage-collector.md index 334a09ce6a..7ba3dcb213 100644 --- a/docs/src/internals/garbage-collector.md +++ b/docs/src/internals/garbage-collector.md @@ -73,6 +73,47 @@ with the live set, and each has a cheaper exact form: +**Promoted-cohort fulls.** Old-reclaim pacing credits every promotion to its +growth baseline, so old-gen garbage that a minor promoted and that died after +it is invisible to it. A document parse loop is that shape: each result's +top-level array is born old, its young contents stay reachable through that +array's remembered slots until a full proves the array dead, and every +nursery minor promotes the previous (dead) tree together with the current +one. A full therefore also becomes due when the bytes promoted since the last +full reach the larger of one base nursery and the old-gen live set the last +full verified. The bound is consulted only right after a nursery minor at a +precise safepoint, so the full runs with precise roots and, after an in-place +promotion, with an empty young generation; it is not an arm of the old-reclaim +growth predicate. A cohort full that reclaims less than half its cohort +doubles the bound (at most three times) and a productive one restores it, so a +heap whose promoted data stays live pays a logarithmic number of such fulls. +When the minor's promotion can reach the bound, its promotion walk records +each promoted block's census facts, and the full started at the same +safepoint adopts them instead of walking those blocks again. `PERRY_GC_DIAG=1` +prints `promoted_since_full=`/`cohort_bound=` on `[gc-trigger]` lines and a +`[gc-promoted-cohort]` line per cohort full. + + + + +**One pass per block in a synchronous full.** An unbudgeted census and an +unbudgeted sweep parse each arena block themselves instead of calling the +object cursor once per object, with the block's constants hoisted; both are +compared with the per-object walk in tests. The hole-list rebuild skips a live +old block the census proved free of invalidated headers. A full no longer +expands promoted page runs up front: a page's run is expanded only right before +the sweep invalidates a dead header on it. The mark reads its proxy and +weak-holder facts once per object, finds a pointer's census block through a +direct-mapped 1 MiB window index, and marks a pointer-free object that is not a +forwarding stub without queueing it. + + + + + + + + `PERRY_GC_SCAVENGE` is on by default and lets nursery pressure route to the direct minor. `PERRY_GC_SCAVENGE_NURSERY_MB` tunes its base high-water cap, 16 MiB by default From bac8c66fe31300beb15d9a84163b140edde5cb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 02:56:03 +0200 Subject: [PATCH 26/40] gc: the hole-list rebuild filters blocks in place instead of copying the liveness vector (#10182) The hole-free skip copied block_has_live and kept a second per-block vector for the sweep's invalidations. The rebuild now takes a block filter, and an invalidation clears the block's hole-free fact directly. --- crates/perry-runtime/src/gc/old_free.rs | 29 ++++++++------- .../src/gc/oldgen/sweep_objects.rs | 37 ++++++++----------- 2 files changed, 31 insertions(+), 35 deletions(-) diff --git a/crates/perry-runtime/src/gc/old_free.rs b/crates/perry-runtime/src/gc/old_free.rs index 48efe7daeb..209358fd1f 100644 --- a/crates/perry-runtime/src/gc/old_free.rs +++ b/crates/perry-runtime/src/gc/old_free.rs @@ -98,6 +98,14 @@ pub(super) fn old_free_rebuild_from_live_old_blocks( block_has_live: &[bool], old_block_start: usize, ) { + old_free_rebuild_from_old_blocks(|block_idx| { + block_idx >= old_block_start && block_has_live.get(block_idx).copied().unwrap_or(false) + }); +} + +/// [`old_free_rebuild_from_live_old_blocks`] over the old blocks `parse` +/// selects (global block indices). +pub(super) fn old_free_rebuild_from_old_blocks(parse: impl FnMut(usize) -> bool) { OLD_FREE_MAP.with(|m| m.borrow_mut().clear()); OLD_FREE_BYTES.with(|c| c.set(0)); OLD_FREE_NONEMPTY.with(|c| c.set(false)); @@ -105,20 +113,15 @@ pub(super) fn old_free_rebuild_from_live_old_blocks( // (`arena_walk_objects_filtered` and friends) step over invalidated // headers WITHOUT invoking the callback, so a rebuild written against // them silently records zero holes. - crate::arena::old_arena_walk_all_headers_filtered( - |block_idx| { - block_idx >= old_block_start && block_has_live.get(block_idx).copied().unwrap_or(false) - }, - |header_ptr, _block_idx| { - let header = header_ptr as *mut GcHeader; - unsafe { - if (*header).obj_type == 0 { - let total_size = (*header).size as usize; - old_free_push(header as usize + GC_HEADER_SIZE, total_size); - } + crate::arena::old_arena_walk_all_headers_filtered(parse, |header_ptr, _block_idx| { + let header = header_ptr as *mut GcHeader; + unsafe { + if (*header).obj_type == 0 { + let total_size = (*header).size as usize; + old_free_push(header as usize + GC_HEADER_SIZE, total_size); } - }, - ); + } + }); } /// Take a hole of exactly `total_size` bytes, if one exists. When diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs index 23cdc39108..76bc428f86 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs @@ -63,9 +63,8 @@ pub(super) struct ArenaSweepObjectsState { /// #10182: per block, the census parsed every header and none of them was /// invalidated, and the block has not changed since. Empty unless the /// block skip ran against an armed census. + /// Cleared for a block as soon as this sweep invalidates a header in it. census_hole_free: Vec, - /// #10182: per block, this sweep invalidated a dead header in it. - invalidated_in_block: Vec, } impl ArenaSweepObjectsState { @@ -107,7 +106,6 @@ impl ArenaSweepObjectsState { block_skip_objects: 0, block_skip_bytes: 0, census_hole_free: Vec::new(), - invalidated_in_block: vec![false; n_blocks], } } @@ -240,28 +238,23 @@ impl ArenaSweepObjectsState { /// re-parses the whole tree to find no hole. pub(super) fn push_live_block_holes(&mut self) { if self.reclaim_dead_old_blocks { - let mut parse = self.block_has_live.clone(); + let old_block_start = self.old_block_start; + let block_has_live = &self.block_has_live; + let hole_free = &self.census_hole_free; let mut skipped = 0u64; - for (block_idx, live) in parse.iter_mut().enumerate() { - if block_idx >= self.old_block_start - && *live - && self - .census_hole_free - .get(block_idx) - .copied() - .unwrap_or(false) - && !self - .invalidated_in_block - .get(block_idx) - .copied() - .unwrap_or(true) + super::old_free_rebuild_from_old_blocks(|block_idx| { + if block_idx < old_block_start + || !block_has_live.get(block_idx).copied().unwrap_or(false) { - *live = false; + return false; + } + if hole_free.get(block_idx).copied().unwrap_or(false) { skipped += 1; + return false; } - } + true + }); super::super::trace::block_skip::note_hole_rebuild_blocks_skipped(skipped); - super::old_free_rebuild_from_live_old_blocks(&parse, self.old_block_start); if crate::gc::gc_diag_enabled() { eprintln!( "[gc-old-free] reusable_bytes={} rebuild_skipped_blocks={skipped}", @@ -634,8 +627,8 @@ impl ArenaSweepObjectsState { #[inline] fn note_invalidated(&mut self, block_idx: usize) { - if let Some(slot) = self.invalidated_in_block.get_mut(block_idx) { - *slot = true; + if let Some(slot) = self.census_hole_free.get_mut(block_idx) { + *slot = false; } } } From c9538770f51b63cdb5e2eb8df829b49c0317a090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 02:56:03 +0200 Subject: [PATCH 27/40] gc: zero the dead stack below a full's frame between the census and the root scan (#10182) A conservative stack scan reads every word from its own stack pointer up, including slots of live frames that no call has written since a deeper frame returned. The one-pass census left a different heap address there than the per-object census did: on records_array_8m:roundtrip every alloc-point full found one more conservative root (8 against 7), kept a dead 7 MB stringify result, lost the 7 MB hole it would have left, and peak RSS rose from 129 to 159 MiB. A full now zeroes 16 KiB of dead stack after building its census; the root counts and the RSS are back to 129 MiB. --- crates/perry-runtime/src/gc/cycle.rs | 20 ++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/scrub_dead_stack.rs | 72 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 crates/perry-runtime/src/gc/tests/scrub_dead_stack.rs diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 024b45ac77..daa0a1fa29 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -867,6 +867,12 @@ impl GcCycleState { trace_phase_record(&mut self.trace, "build_valid_pointer_set", phase_start); return; } + // The census's frames just returned from the stack region the root + // scan's frames are about to occupy, and a conservative stack scan + // reads every word below its caller frames, uninitialized slots + // included. Zero that dead region so a heap address the census walk + // left behind cannot read as a root (#10182). + scrub_dead_stack_below(); let builder = self .valid_builder .take() @@ -1845,5 +1851,19 @@ impl Drop for GcCycleState { } } +/// Zero `DEAD_STACK_SCRUB_WORDS` words of the stack immediately below the +/// caller's frame. The region is dead (below the stack pointer of every live +/// frame), so writing it cannot change program state; it only erases what +/// frames that already returned left there. +#[inline(never)] +pub(super) fn scrub_dead_stack_below() { + let mut words = [0u64; DEAD_STACK_SCRUB_WORDS]; + // Force the zeros to be materialized in this frame. + std::hint::black_box(&mut words); +} + +/// 16 KiB: deeper than the census walk's frames. +const DEAD_STACK_SCRUB_WORDS: usize = 2048; + mod alloc_flag; pub(super) use alloc_flag::restore_minor_in_alloc; diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 40c0dc9dff..d86cbf10b9 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -67,6 +67,7 @@ mod roots; mod runtime_roots; mod scan_fallback; mod schedule; +mod scrub_dead_stack; mod shadow_stack_ops; mod shape_descriptor_authority; mod shape_keys_descriptor_edge; diff --git a/crates/perry-runtime/src/gc/tests/scrub_dead_stack.rs b/crates/perry-runtime/src/gc/tests/scrub_dead_stack.rs new file mode 100644 index 0000000000..04776f48cf --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/scrub_dead_stack.rs @@ -0,0 +1,72 @@ +//! #10182: a synchronous full zeroes the dead stack region below its frame +//! after the census walk and before the root scan (`scrub_dead_stack_below`). +//! +//! A conservative stack scan reads every word from its own stack pointer up, +//! including slots of live frames that were never written since a deeper call +//! returned, so a heap address a returned census frame left there reads as a +//! root. The case plants a sentinel in dead stack exactly where a returned +//! call leaves its frame, and reads that region back from the same depth: +//! without the scrub the sentinel is still there, with it the region is zero. + +use super::super::cycle::scrub_dead_stack_below; + +const SENTINEL: u64 = 0x0000_7fee_dead_beef; +const WORDS: usize = 1024; + +#[inline(never)] +fn plant_sentinels() { + let mut words = [SENTINEL; WORDS]; + std::hint::black_box(&mut words); +} + +#[inline(always)] +fn stack_pointer() -> usize { + let sp: usize; + #[cfg(target_arch = "aarch64")] + unsafe { + std::arch::asm!("mov {}, sp", out(reg) sp); + } + #[cfg(target_arch = "x86_64")] + unsafe { + std::arch::asm!("mov {}, rsp", out(reg) sp); + } + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + { + sp = 0; + } + sp +} + +/// Sentinels left in the `WORDS` words below this frame's stack pointer. +#[inline(never)] +fn sentinels_below_after(scrub: bool) -> usize { + plant_sentinels(); + if scrub { + scrub_dead_stack_below(); + } + let sp = stack_pointer(); + if sp == 0 { + return usize::from(!scrub); + } + (1..=WORDS) + .filter(|&i| { + // SAFETY: the region below the stack pointer belongs to this + // thread's stack mapping; it is dead, not unmapped. + unsafe { std::ptr::read_volatile((sp - i * 8) as *const u64) == SENTINEL } + }) + .count() +} + +#[test] +fn the_scrub_erases_what_a_returned_frame_left_below_the_stack_pointer() { + let left = sentinels_below_after(false); + assert!( + left > 0, + "premise: a returned frame leaves its words in dead stack" + ); + assert_eq!( + sentinels_below_after(true), + 0, + "the scrub must zero the dead region the next call chain will occupy" + ); +} From 5fedc56201723102775128f915861f3976a9b75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 03:23:10 +0200 Subject: [PATCH 28/40] gc: re-pin the census window after the dead-stack scrub (#10182) --- scripts/gc_runtime_root_holders.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index f49a7563a6..ddf62cadff 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -313,7 +313,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes — in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -329,7 +329,7 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "5c151725460ffb92a55a6bee781123ef5159263b4ce5958d16570f78216e0d67", - "crates/perry-runtime/src/gc/cycle.rs": "42c1654ab4ff7886da98ac36589a44080e0e10132460d245643cb14309d0ac80", + "crates/perry-runtime/src/gc/cycle.rs": "9ebee6df0040a50a7f24bd098ce042512a93a1a7ac232346b95fe66cfdf9c0fd", "crates/perry-runtime/src/gc/mod.rs": "9fedd2790f48154aaeceefb4805d3fbaa2fdf3c407529b326425fde86c2bf9a5", "crates/perry-runtime/src/gc/policy.rs": "d588a0a3ffe21f8523d419fdced0942ca36b63c4d6c38faac93db1042304f8a9", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" From b2f41fd9121e9b28722395ac2bdb577b92184020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 04:03:37 +0200 Subject: [PATCH 29/40] changelog: pacing-full cost and promoted-cohort fulls (#10182) --- changelog.d/10241-pacing-full-cost.md | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 changelog.d/10241-pacing-full-cost.md diff --git a/changelog.d/10241-pacing-full-cost.md b/changelog.d/10241-pacing-full-cost.md new file mode 100644 index 0000000000..a472adeedc --- /dev/null +++ b/changelog.d/10241-pacing-full-cost.md @@ -0,0 +1,77 @@ +A synchronous full over a heap of promoted JSON trees costs about half as much, +and a full can now be paced by the bytes promoted since the previous full +(#10182). Stacked on #10220. + +**Cheaper full** (each change has a planted test and a sabotaged twin): + +- **Runs expanded only where the sweep reshapes.** `GcCycleState::new_full` + no longer expands every described promoted page run. + `PendingOldUnregister::defer` now expands the runs on a dead object's pages + before it invalidates that object. +- **One-pass census.** An unbounded census hands each arena block to + `census_whole_block`, which parses headers and start bits in one loop. + - A block the preceding minor stamped while promoting it in place is adopted + from a record built during that stamping (`gc/trace/adopt_census.rs`), + without being read again. + - Adoption happens only in a promoted-cohort full that follows that minor + at the same safepoint. + - A census block is found through a direct-mapped 1 MiB window index. +- **One-pass sweep and fewer hole scans.** An unbudgeted sweep walks each + block in one pass. The hole-list rebuild skips live blocks the census proved + hole-free, filtering the block list in place. +- **Cheaper mark.** The mark reads its per-object facts (proxy tracing, weak + holder) once per object. It sets a pointer-free object's mark bit without + queueing it. It prefetches ahead in the worklist and before a range + descriptor's children. `GcMutableSlot` classifies `external` lazily. +- **Stack scrub.** Between the census and the root scan, a full zeroes 16 KiB + of dead stack below its own frame. Census residue there had become one extra + conservative root. That root kept a dead 7 MB string alive and cost + `records_array_8m:roundtrip` +30 MiB peak RSS. + +On #10220's probe (one 20 MB tree in the old generation, `gc()` in a loop), a +steady full drops from 38 ms to 25 ms: census 4.9 → 2.4 ms, mark 25 → 18 ms, +sweep 7.2 → 3.3 ms. + +The probe's pacing full drops from 80 ms to 27 ms: rebuild 23 → 0 ms, sweep +21 → 4.4 ms. On base that full fires at an allocation point, with a forced +conservative scan. Here it is a promoted-cohort full at the safepoint, and it +fires earlier. The `gc()` that follows therefore sweeps more, but still costs +less than base's: 42 ms against 47 ms. + +**Promoted-cohort fulls** (`gc/promoted_cohort.rs`): + +- **Trigger.** A copying minor at a precise safepoint runs a full right after + it when the bytes promoted since the last full reach + `max(nursery cap, old live at the last full << shift)`. That full uses the + safepoint's roots, with no forced conservative scan. +- **Backoff.** A full that reclaims less than half its cohort raises `shift` + (capped at 3). A productive full resets it to 0. +- **Old-reclaim baseline untouched.** The cohort does not change that + baseline, so both baseline tests #10204 broke still pass. +- **Diagnostics.** `PERRY_GC_DIAG=1` prints `[gc-promoted-cohort]` and adds + `promoted_since_full=` / `cohort_bound=` to `[gc-trigger]`. + +On a real cohort full in `records_array_20m:parse` the pause is 30–32 ms, down +from the 62–68 ms #10220 measured. + +**This does not meet #10182's acceptance bar.** Interleaved best-of-3 on the +same tree, base `9a05821b9e`: + +| row | base | this branch | best node/bun | +|---|---|---|---| +| `records_array_20m:parse` | 143.8 ms / 240 MiB | 209.6 / 167 | 208.0 / 220 | +| `records_array_20m:scan` | 153.7 / 240 | 218.4 / 167 | 212.0 / 225 | +| `records_array_20m:sparse` | 145.5 / 240 | 214.7 / 167 | 208.4 / 220 | +| `records_object_20m:parse` | 146.3 / 240 | 210.9 / 167 | 207.4 / 220 | +| `records_array_20m:roundtrip` | 105.0 / 260 | 117.2 / 231 | 162.1 / 261 | +| `records_array_8m:scan` | 182.0 / 189 | 227.1 / 127 | 188.7 / 110 | + +- **Target rows.** All four now beat node/bun on RSS but are 1–3 % over the + best CPU. Each iteration promotes its dead predecessor's tree, so each row + runs three fulls of about 22 ms net against a lead of about 62 ms. +- **Other rows.** `records_array_8m:scan` loses its CPU lead (4 cohort fulls), + and `records_array_20m:roundtrip` is +12 % CPU. +- **Mechanism commits alone** (cohort trigger disabled locally): no row is + outside ±2 % CPU in both of two interleaved matrices. The 20 MB rows read + +0.0 to +1.8 %. `records_array_8m:parse`/`sparse` peak RSS falls from + 109 MiB to 97/98 MiB. From 710348ad8433e085b12c3ab6bd9322e2afb3ede9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 04:08:45 +0200 Subject: [PATCH 30/40] changelog: state the target-row CPU miss exactly (#10182) --- changelog.d/10241-pacing-full-cost.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/10241-pacing-full-cost.md b/changelog.d/10241-pacing-full-cost.md index a472adeedc..4ee44cc8cb 100644 --- a/changelog.d/10241-pacing-full-cost.md +++ b/changelog.d/10241-pacing-full-cost.md @@ -66,9 +66,9 @@ same tree, base `9a05821b9e`: | `records_array_20m:roundtrip` | 105.0 / 260 | 117.2 / 231 | 162.1 / 261 | | `records_array_8m:scan` | 182.0 / 189 | 227.1 / 127 | 188.7 / 110 | -- **Target rows.** All four now beat node/bun on RSS but are 1–3 % over the - best CPU. Each iteration promotes its dead predecessor's tree, so each row - runs three fulls of about 22 ms net against a lead of about 62 ms. +- **Target rows.** All four now beat node/bun on RSS but are +0.8 to +3.0 % + over the best CPU. Each iteration promotes its dead predecessor's tree, so + each row runs three fulls of about 22 ms net against a lead of 58–64 ms. - **Other rows.** `records_array_8m:scan` loses its CPU lead (4 cohort fulls), and `records_array_20m:roundtrip` is +12 % CPU. - **Mechanism commits alone** (cohort trigger disabled locally): no row is From edec7cc5c62c2bc19c794bf1ba78e74caec53621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 07:11:09 +0200 Subject: [PATCH 31/40] gc: a cohort full feeds the survival of the blocks its own minor promoted (#10182) A promoted-cohort full resets the untraced-promotion budget, so a workload that turns from building a live set to churning never re-measures: every minor promotes the churn untraced and every cohort full marks the whole live set to reclaim it (14_grow_then_churn: 13 cohort fulls, 0 copied). The full's sweep now measures, block by block, how much of what the minor at the same safepoint promoted is still reachable (the blocks the promotion walk recorded for census adoption). Below the in-place promotion threshold that figure replaces the young-survival predictor, so the next minor evacuates and measures instead of promoting on faith. A parse loop measures ~1000 there (the last minor promoted the tree being parsed) and keeps its untraced promotions. --- .../src/gc/oldgen/sweep_objects.rs | 22 +++ crates/perry-runtime/src/gc/policy.rs | 25 ++- .../perry-runtime/src/gc/promote_in_place.rs | 33 ++++ .../perry-runtime/src/gc/promoted_cohort.rs | 179 ++++++++++++++++++ .../src/gc/tests/promoted_cohort.rs | 158 ++++++++++++++++ .../src/gc/trace/adopt_census.rs | 13 ++ 6 files changed, 428 insertions(+), 2 deletions(-) diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs index 76bc428f86..b3a5628669 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs @@ -65,6 +65,9 @@ pub(super) struct ArenaSweepObjectsState { /// block skip ran against an armed census. /// Cleared for a block as soon as this sweep invalidates a header in it. census_hole_free: Vec, + /// #10241: a promoted-cohort full armed `promoted_cohort`'s survival + /// probe, and this is the synchronous full sweep that answers it. + survival_probe: bool, } impl ArenaSweepObjectsState { @@ -106,6 +109,7 @@ impl ArenaSweepObjectsState { block_skip_objects: 0, block_skip_bytes: 0, census_hole_free: Vec::new(), + survival_probe: false, } } @@ -128,6 +132,10 @@ impl ArenaSweepObjectsState { /// A skipped block contributes nothing to `block_has_live`, which is the /// only liveness the cleanup reads. pub(super) fn apply_block_skip(&mut self, census: &super::super::trace::BlockCensus) { + // Reached only from a synchronous full sweep, which is the one whose + // marks are final and whose whole-block walk the probe reads. + self.survival_probe = + !self.minor_sweep && super::super::promoted_cohort::survival_probe_armed(); if self.minor_sweep || !self.reclaim_dead_old_blocks || self.targeted_old_blocks.is_some() @@ -192,6 +200,12 @@ impl ArenaSweepObjectsState { } skip[block_idx] = true; any = true; + if self.survival_probe { + super::super::promoted_cohort::note_probe_block_skipped( + snapshot.data, + snapshot.offset, + ); + } self.freed_bytes = self.freed_bytes.saturating_add(block.bytes); if block_idx < self.resettable_general_n { self.eden_dead_bytes = self.eden_dead_bytes.saturating_add(block.bytes); @@ -269,8 +283,16 @@ impl ArenaSweepObjectsState { let mut done = false; if budget == usize::MAX && self.cursor.at_block_boundary() { while let Some((block_idx, data, offset, size)) = self.cursor.next_whole_block() { + let live_before = self.arena_live_bytes; // SAFETY: the block was snapshotted by this sweep's cursor. unsafe { self.sweep_whole_block(block_idx, data, offset, size) }; + if self.survival_probe { + super::super::promoted_cohort::note_probe_block_swept( + data, + offset, + self.arena_live_bytes - live_before, + ); + } } remaining = 0; done = true; diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index f40148bd57..2498583f31 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -3579,20 +3579,41 @@ pub(super) fn run_promoted_cohort_full_if_due() -> bool { super::diag_sites::trigger_decision("safepoint", "PromotedCohort"); super::diag_sites::set_full_site("safepoint_promoted_cohort"); let adopted_before = super::trace::adopt_census::adopted_blocks(); + // #10241: the full's sweep measures how much of what the minor at this + // safepoint promoted is still reachable (`promoted_cohort::PromotedSurvival`). + super::promoted_cohort::arm_survival_probe(super::trace::adopt_census::ready_blocks()); super::trace::adopt_census::begin_adopting(); // No `force_full_scan`: roots are precise at this safepoint. gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::OldGenBytes)) .emit_after_current(); super::trace::adopt_census::discard(); + let survival = super::promoted_cohort::take_survival_probe(); + #[cfg(test)] + let feed = !super::promoted_cohort::survival_sabotage::unfed(); + #[cfg(not(test))] + let feed = true; + let survival_permille = survival.as_ref().and_then(|s| { + if feed { + super::note_full_measured_promotion_survival(s.promoted_bytes, s.live_bytes) + } else { + s.permille() + } + }); let adopted = super::trace::adopt_census::adopted_blocks() - adopted_before; let after = old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); let reclaimed = before.saturating_sub(after); let productive = super::promoted_cohort::record_full_yield(cohort, reclaimed); if super::gc_diag_enabled() { + let (blocks, promoted_by_minor, live_of_promoted) = survival + .as_ref() + .map_or((0, 0, 0), |s| (s.blocks, s.promoted_bytes, s.live_bytes)); eprintln!( "[gc-promoted-cohort] full cohort={cohort} bound={bound} reclaimed={reclaimed} \ - productive={productive} adopted_census_blocks={adopted} backoff_shift={}", - super::promoted_cohort::backoff_shift() + productive={productive} adopted_census_blocks={adopted} backoff_shift={} \ + promoted_blocks={blocks} promoted_by_minor={promoted_by_minor} \ + live_of_promoted={live_of_promoted} survival_permille={}", + super::promoted_cohort::backoff_shift(), + survival_permille.map_or_else(|| "none".to_string(), |p| p.to_string()) ); } true diff --git a/crates/perry-runtime/src/gc/promote_in_place.rs b/crates/perry-runtime/src/gc/promote_in_place.rs index 5f2fe7d5b0..2e45608f05 100644 --- a/crates/perry-runtime/src/gc/promote_in_place.rs +++ b/crates/perry-runtime/src/gc/promote_in_place.rs @@ -560,6 +560,39 @@ pub(super) fn note_full_collection_reclaimed_old_gen() { OLD_GEN_AT_LAST_MEASUREMENT.with(|c| c.set(crate::arena::old_gen_in_use_bytes())); } +/// A promoted-cohort full measured, with its own mark, that `live_bytes` of the +/// `promoted_bytes` the minor at the same safepoint promoted survived +/// (#10241, `promoted_cohort::PromotedSurvival`). +/// +/// That is a young-survival measurement of the same generation the predictor +/// describes, taken one safepoint later by a trace that follows every edge, so +/// a figure under [`PROMOTE_SURVIVAL_THRESHOLD_PERMILLE`] replaces the ratio the +/// promotion was admitted on: the next minor evacuates and measures again +/// instead of promoting the churn on faith. A figure at or above it leaves the +/// predictor alone — it confirms the promotion, and the minor's own +/// measurement, where it has one, is the finer figure for untraced admission. +/// +/// Why not force a measuring minor after every reclaiming full instead: on a +/// parse loop whose trees die one tree later every cohort full reclaims its +/// whole cohort, and a traced minor over a 58 MB young tree costs ~100 ms +/// against ~15 ms untraced (`records_array_20m:parse`). This figure is ~1000 +/// there, because the last minor promoted the tree still being parsed. +pub(super) fn note_full_measured_promotion_survival( + promoted_bytes: usize, + live_bytes: usize, +) -> Option { + let permille = super::promoted_cohort::PromotedSurvival { + blocks: 0, + promoted_bytes, + live_bytes, + } + .permille()?; + if permille < PROMOTE_SURVIVAL_THRESHOLD_PERMILLE { + LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(Some(permille))); + } + Some(permille) +} + /// How many cycles promoted in place, and how many objects they promoted. /// The "did the subject actually run?" counters — a green benchmark that never /// entered the path proves nothing. diff --git a/crates/perry-runtime/src/gc/promoted_cohort.rs b/crates/perry-runtime/src/gc/promoted_cohort.rs index 28bc6e18b7..8f190bb0ef 100644 --- a/crates/perry-runtime/src/gc/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/promoted_cohort.rs @@ -132,6 +132,185 @@ pub(super) fn cohort_fulls() -> u64 { COHORT_FULLS.with(Cell::get) } +/// What a promoted-cohort full's own mark says about the blocks the nursery +/// minor at the same safepoint promoted (#10241). +/// +/// # Why this measurement, and why the cohort's yield is not enough +/// +/// A minor promotes in place, and may skip its trace, on the strength of the +/// PREVIOUS minor's young-survival ratio. An untraced run re-measures only when +/// its byte budget runs out, and every full resets that budget +/// (`note_full_collection_reclaimed_old_gen`). A cohort full every ~`live` +/// promoted bytes therefore keeps a workload that turned from building a live +/// set to churning on the untraced path indefinitely: nothing ever measures the +/// churn, every minor promotes it, and every cohort full marks the whole live +/// set to reclaim it (`14_grow_then_churn`: 13 cohort fulls, 0 copied objects). +/// +/// The cohort's yield cannot tell that apart from a parse loop whose promoted +/// trees die one tree later: both reclaim about the whole cohort. What differs +/// is the survival of the blocks the LAST minor promoted. On a parse loop they +/// hold the tree still being built and the tail of the previous one, both +/// reachable; on a churn phase they are the churn, dead by the full. The full's +/// mark is exact (full reachability, no remembered-set conservatism), and the +/// sweep reads it block by block anyway, so the figure costs one map lookup per +/// swept block of the cohort full and nothing anywhere else. +/// +/// # Exact or nothing +/// +/// The blocks are the ones the minor's promotion walk recorded for census +/// adoption (`trace::adopt_census`), keyed by data address with the bump extent +/// and header bytes they held at promotion. Each is accounted when the full's +/// sweep either walks it whole at the same extent (its live bytes are the +/// sweep's own `arena_live_bytes` delta) or reclaims it unwalked as a dead +/// block (live 0). A block the sweep reaches any other way, at another extent, +/// or not at all leaves the measurement unset. +pub(super) struct PromotedSurvival { + pub(super) blocks: usize, + pub(super) promoted_bytes: usize, + pub(super) live_bytes: usize, +} + +impl PromotedSurvival { + pub(super) fn permille(&self) -> Option { + (self.promoted_bytes > 0).then(|| { + (self.live_bytes as u64) + .saturating_mul(1000) + .checked_div(self.promoted_bytes as u64) + .unwrap_or(0) + .min(1000) + }) + } +} + +struct SurvivalProbe { + /// Recorded blocks not yet accounted: data address -> (extent, bytes). + pending: crate::fast_hash::PtrHashMap, + blocks: usize, + promoted_bytes: u64, + live_bytes: u64, + exact: bool, +} + +crate::perry_thread_local! { + /// The armed probe of the cohort full in progress, if any. Holds arena + /// block data addresses only as identity keys for the sweep's snapshot; + /// nothing is read through them. + static SURVIVAL_PROBE: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +/// Arm the probe over `(data, extent, bytes)` blocks before the cohort full. +pub(super) fn arm_survival_probe(blocks: Vec<(usize, usize, u64)>) { + let mut pending = crate::fast_hash::new_ptr_hash_map(); + let mut promoted_bytes = 0u64; + for (data, extent, bytes) in blocks { + promoted_bytes = promoted_bytes.saturating_add(bytes); + pending.insert(data, (extent, bytes)); + } + let probe = (!pending.is_empty()).then(|| SurvivalProbe { + blocks: pending.len(), + pending, + promoted_bytes, + live_bytes: 0, + exact: true, + }); + SURVIVAL_PROBE.with(|p| *p.borrow_mut() = probe); +} + +/// Read once per synchronous full sweep. +pub(super) fn survival_probe_armed() -> bool { + SURVIVAL_PROBE.with(|p| p.borrow().is_some()) +} + +/// The sweep walked the block at `data` whole, to `extent`, and kept `live` +/// bytes of it. +pub(super) fn note_probe_block_swept(data: usize, extent: usize, live: u64) { + account_probe_block(data, extent, live); +} + +/// The sweep reclaimed the block at `data` without walking it: nothing in it +/// was reached. +pub(super) fn note_probe_block_skipped(data: usize, extent: usize) { + account_probe_block(data, extent, 0); +} + +fn account_probe_block(data: usize, extent: usize, live: u64) { + SURVIVAL_PROBE.with(|p| { + let mut probe = p.borrow_mut(); + let Some(probe) = probe.as_mut() else { + return; + }; + let Some((recorded_extent, _)) = probe.pending.remove(&data) else { + return; + }; + if recorded_extent != extent { + probe.exact = false; + } + probe.live_bytes = probe.live_bytes.saturating_add(live); + }); +} + +/// Disarm the probe; the measurement when every recorded block was accounted. +pub(super) fn take_survival_probe() -> Option { + let probe = SURVIVAL_PROBE.with(|p| p.borrow_mut().take())?; + let survival = (probe.exact && probe.pending.is_empty()).then(|| PromotedSurvival { + blocks: probe.blocks, + promoted_bytes: usize::try_from(probe.promoted_bytes).unwrap_or(usize::MAX), + live_bytes: usize::try_from(probe.live_bytes).unwrap_or(usize::MAX), + }); + #[cfg(test)] + LAST_SURVIVAL_FOR_TESTS.with(|c| { + c.set( + survival + .as_ref() + .map(|s| (s.blocks, s.promoted_bytes, s.live_bytes)), + ) + }); + survival +} + +#[cfg(test)] +thread_local! { + static LAST_SURVIVAL_FOR_TESTS: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + +/// `(blocks, promoted bytes, live bytes)` of the last probe taken, when it +/// measured (tests only). +#[cfg(test)] +pub(super) fn last_survival_for_tests() -> Option<(usize, usize, usize)> { + LAST_SURVIVAL_FOR_TESTS.with(std::cell::Cell::get) +} + +/// Sabotage switch for the survival tests: the cohort full measures but does +/// not feed the promotion predictor. Test builds only. +#[cfg(test)] +pub(super) mod survival_sabotage { + use std::cell::Cell; + + thread_local! { + static UNFED: Cell = const { Cell::new(false) }; + } + + pub(in crate::gc) fn unfed() -> bool { + UNFED.with(Cell::get) + } + + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(UNFED.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + UNFED.with(|s| s.set(self.0)); + } + } +} + /// Sabotage switch for the cohort tests: every cohort full counts as /// productive, so the bound never backs off. Test builds only. #[cfg(test)] diff --git a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs index deb1984774..9e933308bf 100644 --- a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs @@ -5,6 +5,11 @@ //! arm (the two baseline tests #10204 broke keep their meaning), and one real //! collection: a promoting minor credits the cohort, and the cohort full that //! follows it reclaims a promoted object that died. +//! +//! #10241: the cohort full measures the survival of what the minor at its own +//! safepoint promoted, and a dead same-safepoint cohort turns the next minor +//! back into an evacuating one (with a sabotaged twin that does not feed the +//! predictor, and a live cohort that leaves it alone). use super::super::policy::{ credit_promoted_bytes_to_old_baseline, old_reclaim_pressure_due, @@ -184,3 +189,156 @@ fn below_the_bound_no_cohort_full_runs_and_the_dead_promoted_object_stays() { "only a full can reclaim a promoted object" ); } + +/// Outcome of one untraced promotion, a cohort full at the same safepoint, and +/// the minor after it. +struct SurvivalOutcome { + /// `(blocks, promoted bytes, live bytes)` the full measured. + measured: Option<(usize, usize, usize)>, + predictor_after_full: Option, + next_minor_in_place: bool, + next_minor_untraced: bool, +} + +/// Root an array of `count` young strings in `slot`; returns the first string. +fn rooted_young_strings(slot: u32, count: usize) -> usize { + let mut array = crate::array::js_array_alloc(count as u32); + let first = young_leaf(); + array = crate::array::js_array_push_jsvalue(array, string_bits(first)); + for _ in 1..count { + array = crate::array::js_array_push_jsvalue(array, string_bits(young_leaf())); + } + js_shadow_slot_set(slot, ptr_bits(array as usize)); + first +} + +/// A young population promoted whole and untraced by a minor that records its +/// blocks, then the cohort full at the same safepoint — with that population +/// still rooted (`live`) or dropped — then a minor over a fresh rooted +/// population. `fed == false` arms the sabotage that keeps the full's +/// measurement away from the predictor. +fn promote_then_cohort_full_then_minor(live: bool, fed: bool) -> SurvivalOutcome { + use super::super::trace::adopt_census; + let _guard = CopyingNurseryTestGuard::new(4); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _promote = super::super::InPlacePromotionTestGuard::untraced(); + cohort::seed_for_tests(0, 0, 0); + + const COUNT: usize = 6000; + let probe_leaf = rooted_young_strings(0, COUNT); + assert!( + crate::arena::pointer_in_nursery(probe_leaf), + "premise: young population" + ); + let untraced_before = untraced_promotion_cycles(); + adopt_census::begin_recording(); + let trace = collect_minor_trace(GcTriggerKind::Direct); + adopt_census::finish_recording(); + assert!( + trace.copying_nursery.in_place_promotion, + "premise: the minor promoted in place" + ); + assert_eq!( + untraced_promotion_cycles() - untraced_before, + 1, + "premise: the promotion skipped the trace" + ); + if !live { + js_shadow_slot_set(0, crate::value::TAG_UNDEFINED); + } + cohort::seed_for_tests(cohort::bound_bytes(), 0, 0); + let ran = { + let _sabotage = (!fed).then(cohort::survival_sabotage::Guard::arm); + run_promoted_cohort_full_if_due() + }; + adopt_census::discard(); + assert!(ran, "premise: the cohort full ran"); + let measured = cohort::last_survival_for_tests(); + let predictor_after_full = super::super::last_young_survival_permille(); + + rooted_young_strings(1, COUNT); + let untraced_before = untraced_promotion_cycles(); + let next = collect_minor_trace(GcTriggerKind::Direct); + let outcome = SurvivalOutcome { + measured, + predictor_after_full, + next_minor_in_place: next.copying_nursery.in_place_promotion, + next_minor_untraced: untraced_promotion_cycles() - untraced_before == 1, + }; + js_shadow_slot_set(0, crate::value::TAG_UNDEFINED); + js_shadow_slot_set(1, crate::value::TAG_UNDEFINED); + cohort::seed_for_tests(0, 0, 0); + outcome +} + +fn permille((_, promoted, live): (usize, usize, usize)) -> u64 { + (live as u64 * 1000 / promoted as u64).min(1000) +} + +#[test] +fn a_dead_same_safepoint_cohort_turns_the_next_minor_into_an_evacuation() { + let outcome = promote_then_cohort_full_then_minor(false, true); + let measured = outcome + .measured + .expect("the full's sweep accounted every recorded block"); + assert!( + measured.0 >= 1 && measured.1 > 0, + "the probe measured the promoted blocks: {measured:?}" + ); + let survival = permille(measured); + assert!( + survival < super::super::PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, + "the dropped cohort is dead by the full: {survival} permille ({measured:?})" + ); + assert_eq!( + outcome.predictor_after_full, + Some(survival), + "the full's measurement replaces the ratio the promotion was admitted on" + ); + assert!( + !outcome.next_minor_in_place && !outcome.next_minor_untraced, + "the next minor evacuates and measures instead of promoting on faith" + ); +} + +#[test] +fn sabotaged_unfed_predictor_promotes_the_next_minor_untraced() { + let outcome = promote_then_cohort_full_then_minor(false, false); + let measured = outcome.measured.expect("premise: the probe still measured"); + assert!( + permille(measured) < super::super::PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, + "premise: the cohort is dead by the full ({measured:?})" + ); + assert_eq!( + outcome.predictor_after_full, + Some(1000), + "unfed, the predictor keeps the ratio the dead cohort was promoted on" + ); + assert!( + outcome.next_minor_in_place && outcome.next_minor_untraced, + "without the feed the next minor promotes untraced again — the \ + 14_grow_then_churn regime, where no minor ever measures the churn" + ); +} + +#[test] +fn a_live_same_safepoint_cohort_leaves_the_predictor_at_retained() { + let outcome = promote_then_cohort_full_then_minor(true, true); + let measured = outcome + .measured + .expect("the full's sweep accounted every recorded block"); + let survival = permille(measured); + assert!( + survival >= super::super::PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, + "the rooted cohort survives the full: {survival} permille ({measured:?})" + ); + assert_eq!( + outcome.predictor_after_full, + Some(1000), + "a confirming measurement leaves the predictor alone" + ); + assert!( + outcome.next_minor_in_place && outcome.next_minor_untraced, + "a retained cohort keeps the next minor on the untraced promotion" + ); +} diff --git a/crates/perry-runtime/src/gc/trace/adopt_census.rs b/crates/perry-runtime/src/gc/trace/adopt_census.rs index 292ecf1f47..d0005c9758 100644 --- a/crates/perry-runtime/src/gc/trace/adopt_census.rs +++ b/crates/perry-runtime/src/gc/trace/adopt_census.rs @@ -106,6 +106,19 @@ pub(crate) fn discard() { STATE.with(|s| *s.borrow_mut() = State::Off); } +/// `(data, extent, bytes)` of every block the minor at this safepoint recorded +/// and a full has not started adopting yet. Read before `begin_adopting`: the +/// census removes a record as it adopts it. +pub(crate) fn ready_blocks() -> Vec<(usize, usize, u64)> { + STATE.with(|s| match &*s.borrow() { + State::Ready(map) => map + .iter() + .map(|(&data, block)| (data, block.extent, block.bytes)) + .collect(), + _ => Vec::new(), + }) +} + /// Take the record of the block at `data`, if the census may adopt one. fn take(data: usize) -> Option { STATE.with(|s| match &mut *s.borrow_mut() { From 1e6c3ef9c0d3b26f8d7a4f9f26a00a4705f4e4cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 07:33:38 +0200 Subject: [PATCH 32/40] gc: feed the cohort full's survival only when it is the minor's view (#10182) records_array_20m:parse measures 500 permille over the blocks its last minor promoted, not ~1000: that minor promotes the dead previous tree with the live current one, because the dead tree's born-old top array keeps its records reachable through remembered slots. Fed to the predictor, that made the next minor evacuate and copy both trees (~95 ms, +50 MB RSS). A minor measures what it reaches from roots and from every old object on its dirty pages, so the full's survival equals the minor's exactly when no object on those pages that the full's mark left unmarked holds a dirty slot into the promoted blocks. The minor now notes its remembered set (page keys only) while the promotion census records; the full checks those parents after its mark, before its sweep, and the predictor is fed only when the view is exact. --- crates/perry-runtime/src/gc/barrier/mod.rs | 2 +- crates/perry-runtime/src/gc/copying.rs | 3 + crates/perry-runtime/src/gc/cycle.rs | 3 + .../src/gc/oldgen/sweep_objects.rs | 6 +- crates/perry-runtime/src/gc/policy.rs | 27 +- .../perry-runtime/src/gc/promote_in_place.rs | 35 +- .../perry-runtime/src/gc/promoted_cohort.rs | 183 +------- .../src/gc/promoted_cohort/survival.rs | 423 ++++++++++++++++++ .../src/gc/tests/promoted_cohort.rs | 11 +- .../src/gc/trace/adopt_census.rs | 6 +- 10 files changed, 482 insertions(+), 217 deletions(-) create mode 100644 crates/perry-runtime/src/gc/promoted_cohort/survival.rs diff --git a/crates/perry-runtime/src/gc/barrier/mod.rs b/crates/perry-runtime/src/gc/barrier/mod.rs index 7a78753ca2..1b4e1a2725 100644 --- a/crates/perry-runtime/src/gc/barrier/mod.rs +++ b/crates/perry-runtime/src/gc/barrier/mod.rs @@ -220,7 +220,7 @@ unsafe fn process_dirty_slot_work( *changed |= *slot != before; } -fn dirty_slot_ranges_for( +pub(super) fn dirty_slot_ranges_for( range: HeapSlotRange, dirty_pages: &crate::fast_hash::PtrHashSet, stats: &mut RememberedSetTraceStats, diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 8cfd1424cd..8ff4869d9a 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -1353,6 +1353,9 @@ pub(super) fn run_copied_minor_attempt( // cycle — a missing-edge bug one collection later. let remembered_phase_start = PhaseDiag::start(&phase_diag); let snapshot = remembered_dirty_snapshot(); + // #10241: a cohort full at this safepoint asks whether a dead parent in + // this remembered set held what this minor promotes. + super::promoted_cohort::survival::note_minor_remembered_parents(&snapshot); // #9754: objects whose every slot the dirty scan visited in-body — the // post-cycle coverage restore skips them (see `scan_dirty_object_slots`). // #9835: this set is rebuilt from EMPTY on every minor and reaches ~1,000 diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index daa0a1fa29..888d07bffe 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1503,6 +1503,9 @@ impl GcCycleState { // `PERRY_GC_CENSUS` pass 2: marks are final and nothing is // swept yet; only synchronous full cycles are exact. super::census::census_take_if_armed_at_full_sweep_start(); + // #10241: same point, for a promoted-cohort full's survival + // probe (a no-op unless one is armed). + super::promoted_cohort::survival::check_minor_view_at_full_sweep_start(); } let (do_age_bump, reclaim_dead_old_blocks, targeted_old_blocks, sweep_malloc) = diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs index b3a5628669..9ca401ca4c 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs @@ -135,7 +135,7 @@ impl ArenaSweepObjectsState { // Reached only from a synchronous full sweep, which is the one whose // marks are final and whose whole-block walk the probe reads. self.survival_probe = - !self.minor_sweep && super::super::promoted_cohort::survival_probe_armed(); + !self.minor_sweep && super::super::promoted_cohort::survival::survival_probe_armed(); if self.minor_sweep || !self.reclaim_dead_old_blocks || self.targeted_old_blocks.is_some() @@ -201,7 +201,7 @@ impl ArenaSweepObjectsState { skip[block_idx] = true; any = true; if self.survival_probe { - super::super::promoted_cohort::note_probe_block_skipped( + super::super::promoted_cohort::survival::note_probe_block_skipped( snapshot.data, snapshot.offset, ); @@ -287,7 +287,7 @@ impl ArenaSweepObjectsState { // SAFETY: the block was snapshotted by this sweep's cursor. unsafe { self.sweep_whole_block(block_idx, data, offset, size) }; if self.survival_probe { - super::super::promoted_cohort::note_probe_block_swept( + super::super::promoted_cohort::survival::note_probe_block_swept( data, offset, self.arena_live_bytes - live_before, diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 2498583f31..347488e4f8 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -3581,19 +3581,19 @@ pub(super) fn run_promoted_cohort_full_if_due() -> bool { let adopted_before = super::trace::adopt_census::adopted_blocks(); // #10241: the full's sweep measures how much of what the minor at this // safepoint promoted is still reachable (`promoted_cohort::PromotedSurvival`). - super::promoted_cohort::arm_survival_probe(super::trace::adopt_census::ready_blocks()); + super::promoted_cohort::survival::arm_survival_probe(super::trace::adopt_census::ready_blocks()); super::trace::adopt_census::begin_adopting(); // No `force_full_scan`: roots are precise at this safepoint. gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::OldGenBytes)) .emit_after_current(); super::trace::adopt_census::discard(); - let survival = super::promoted_cohort::take_survival_probe(); + let survival = super::promoted_cohort::survival::take_survival_probe(); #[cfg(test)] - let feed = !super::promoted_cohort::survival_sabotage::unfed(); + let feed = !super::promoted_cohort::survival::sabotage::unfed(); #[cfg(not(test))] let feed = true; let survival_permille = survival.as_ref().and_then(|s| { - if feed { + if feed && s.minor_view == super::promoted_cohort::survival::MinorView::Exact { super::note_full_measured_promotion_survival(s.promoted_bytes, s.live_bytes) } else { s.permille() @@ -3604,16 +3604,25 @@ pub(super) fn run_promoted_cohort_full_if_due() -> bool { let reclaimed = before.saturating_sub(after); let productive = super::promoted_cohort::record_full_yield(cohort, reclaimed); if super::gc_diag_enabled() { - let (blocks, promoted_by_minor, live_of_promoted) = survival - .as_ref() - .map_or((0, 0, 0), |s| (s.blocks, s.promoted_bytes, s.live_bytes)); + let (blocks, promoted_by_minor, live_of_promoted, minor_view) = + survival.as_ref().map_or((0, 0, 0, "none"), |s| { + ( + s.blocks, + s.promoted_bytes, + s.live_bytes, + s.minor_view.as_str(), + ) + }); eprintln!( "[gc-promoted-cohort] full cohort={cohort} bound={bound} reclaimed={reclaimed} \ productive={productive} adopted_census_blocks={adopted} backoff_shift={} \ promoted_blocks={blocks} promoted_by_minor={promoted_by_minor} \ - live_of_promoted={live_of_promoted} survival_permille={}", + live_of_promoted={live_of_promoted} survival_permille={} minor_view={minor_view} \ + predictor={}", super::promoted_cohort::backoff_shift(), - survival_permille.map_or_else(|| "none".to_string(), |p| p.to_string()) + survival_permille.map_or_else(|| "none".to_string(), |p| p.to_string()), + super::last_young_survival_permille() + .map_or_else(|| "none".to_string(), |p| p.to_string()) ); } true diff --git a/crates/perry-runtime/src/gc/promote_in_place.rs b/crates/perry-runtime/src/gc/promote_in_place.rs index 2e45608f05..5615cb477b 100644 --- a/crates/perry-runtime/src/gc/promote_in_place.rs +++ b/crates/perry-runtime/src/gc/promote_in_place.rs @@ -561,32 +561,29 @@ pub(super) fn note_full_collection_reclaimed_old_gen() { } /// A promoted-cohort full measured, with its own mark, that `live_bytes` of the -/// `promoted_bytes` the minor at the same safepoint promoted survived -/// (#10241, `promoted_cohort::PromotedSurvival`). -/// -/// That is a young-survival measurement of the same generation the predictor -/// describes, taken one safepoint later by a trace that follows every edge, so -/// a figure under [`PROMOTE_SURVIVAL_THRESHOLD_PERMILLE`] replaces the ratio the -/// promotion was admitted on: the next minor evacuates and measures again -/// instead of promoting the churn on faith. A figure at or above it leaves the -/// predictor alone — it confirms the promotion, and the minor's own -/// measurement, where it has one, is the finer figure for untraced admission. +/// `promoted_bytes` the minor at the same safepoint promoted survived — and +/// proved that a minor over that young generation would have measured the same +/// (#10241, `promoted_cohort::survival`, `MinorView::Exact`). The caller feeds +/// nothing else. +/// +/// That is the ratio the predictor describes, taken by a trace at the same +/// safepoint, so a figure under [`PROMOTE_SURVIVAL_THRESHOLD_PERMILLE`] +/// replaces the ratio the promotion was admitted on: the next minor evacuates +/// and measures again instead of promoting the churn on faith. A figure at or +/// above it leaves the predictor alone — it confirms the promotion, and the +/// minor's own measurement is the one untraced admission was priced against. /// /// Why not force a measuring minor after every reclaiming full instead: on a /// parse loop whose trees die one tree later every cohort full reclaims its -/// whole cohort, and a traced minor over a 58 MB young tree costs ~100 ms -/// against ~15 ms untraced (`records_array_20m:parse`). This figure is ~1000 -/// there, because the last minor promoted the tree still being parsed. +/// whole cohort, and a traced minor over a 58 MB young generation costs ~95 ms +/// against ~11 ms untraced (`records_array_20m:parse`). There the full measures +/// 500‰ but a dead born-old tree array refers into the promoted blocks, so a +/// minor would have measured them live and nothing is fed. pub(super) fn note_full_measured_promotion_survival( promoted_bytes: usize, live_bytes: usize, ) -> Option { - let permille = super::promoted_cohort::PromotedSurvival { - blocks: 0, - promoted_bytes, - live_bytes, - } - .permille()?; + let permille = super::promoted_cohort::survival::survival_permille(promoted_bytes, live_bytes)?; if permille < PROMOTE_SURVIVAL_THRESHOLD_PERMILLE { LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(Some(permille))); } diff --git a/crates/perry-runtime/src/gc/promoted_cohort.rs b/crates/perry-runtime/src/gc/promoted_cohort.rs index 8f190bb0ef..b8deb00671 100644 --- a/crates/perry-runtime/src/gc/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/promoted_cohort.rs @@ -45,6 +45,10 @@ use std::cell::Cell; +/// What the cohort full's own mark says about the blocks the minor at its +/// safepoint promoted (#10241). +pub(super) mod survival; + /// A cohort full is productive when it reclaims at least this percentage of /// the cohort it was scheduled for. const PRODUCTIVE_PERCENT: usize = 50; @@ -132,185 +136,6 @@ pub(super) fn cohort_fulls() -> u64 { COHORT_FULLS.with(Cell::get) } -/// What a promoted-cohort full's own mark says about the blocks the nursery -/// minor at the same safepoint promoted (#10241). -/// -/// # Why this measurement, and why the cohort's yield is not enough -/// -/// A minor promotes in place, and may skip its trace, on the strength of the -/// PREVIOUS minor's young-survival ratio. An untraced run re-measures only when -/// its byte budget runs out, and every full resets that budget -/// (`note_full_collection_reclaimed_old_gen`). A cohort full every ~`live` -/// promoted bytes therefore keeps a workload that turned from building a live -/// set to churning on the untraced path indefinitely: nothing ever measures the -/// churn, every minor promotes it, and every cohort full marks the whole live -/// set to reclaim it (`14_grow_then_churn`: 13 cohort fulls, 0 copied objects). -/// -/// The cohort's yield cannot tell that apart from a parse loop whose promoted -/// trees die one tree later: both reclaim about the whole cohort. What differs -/// is the survival of the blocks the LAST minor promoted. On a parse loop they -/// hold the tree still being built and the tail of the previous one, both -/// reachable; on a churn phase they are the churn, dead by the full. The full's -/// mark is exact (full reachability, no remembered-set conservatism), and the -/// sweep reads it block by block anyway, so the figure costs one map lookup per -/// swept block of the cohort full and nothing anywhere else. -/// -/// # Exact or nothing -/// -/// The blocks are the ones the minor's promotion walk recorded for census -/// adoption (`trace::adopt_census`), keyed by data address with the bump extent -/// and header bytes they held at promotion. Each is accounted when the full's -/// sweep either walks it whole at the same extent (its live bytes are the -/// sweep's own `arena_live_bytes` delta) or reclaims it unwalked as a dead -/// block (live 0). A block the sweep reaches any other way, at another extent, -/// or not at all leaves the measurement unset. -pub(super) struct PromotedSurvival { - pub(super) blocks: usize, - pub(super) promoted_bytes: usize, - pub(super) live_bytes: usize, -} - -impl PromotedSurvival { - pub(super) fn permille(&self) -> Option { - (self.promoted_bytes > 0).then(|| { - (self.live_bytes as u64) - .saturating_mul(1000) - .checked_div(self.promoted_bytes as u64) - .unwrap_or(0) - .min(1000) - }) - } -} - -struct SurvivalProbe { - /// Recorded blocks not yet accounted: data address -> (extent, bytes). - pending: crate::fast_hash::PtrHashMap, - blocks: usize, - promoted_bytes: u64, - live_bytes: u64, - exact: bool, -} - -crate::perry_thread_local! { - /// The armed probe of the cohort full in progress, if any. Holds arena - /// block data addresses only as identity keys for the sweep's snapshot; - /// nothing is read through them. - static SURVIVAL_PROBE: std::cell::RefCell> = - const { std::cell::RefCell::new(None) }; -} - -/// Arm the probe over `(data, extent, bytes)` blocks before the cohort full. -pub(super) fn arm_survival_probe(blocks: Vec<(usize, usize, u64)>) { - let mut pending = crate::fast_hash::new_ptr_hash_map(); - let mut promoted_bytes = 0u64; - for (data, extent, bytes) in blocks { - promoted_bytes = promoted_bytes.saturating_add(bytes); - pending.insert(data, (extent, bytes)); - } - let probe = (!pending.is_empty()).then(|| SurvivalProbe { - blocks: pending.len(), - pending, - promoted_bytes, - live_bytes: 0, - exact: true, - }); - SURVIVAL_PROBE.with(|p| *p.borrow_mut() = probe); -} - -/// Read once per synchronous full sweep. -pub(super) fn survival_probe_armed() -> bool { - SURVIVAL_PROBE.with(|p| p.borrow().is_some()) -} - -/// The sweep walked the block at `data` whole, to `extent`, and kept `live` -/// bytes of it. -pub(super) fn note_probe_block_swept(data: usize, extent: usize, live: u64) { - account_probe_block(data, extent, live); -} - -/// The sweep reclaimed the block at `data` without walking it: nothing in it -/// was reached. -pub(super) fn note_probe_block_skipped(data: usize, extent: usize) { - account_probe_block(data, extent, 0); -} - -fn account_probe_block(data: usize, extent: usize, live: u64) { - SURVIVAL_PROBE.with(|p| { - let mut probe = p.borrow_mut(); - let Some(probe) = probe.as_mut() else { - return; - }; - let Some((recorded_extent, _)) = probe.pending.remove(&data) else { - return; - }; - if recorded_extent != extent { - probe.exact = false; - } - probe.live_bytes = probe.live_bytes.saturating_add(live); - }); -} - -/// Disarm the probe; the measurement when every recorded block was accounted. -pub(super) fn take_survival_probe() -> Option { - let probe = SURVIVAL_PROBE.with(|p| p.borrow_mut().take())?; - let survival = (probe.exact && probe.pending.is_empty()).then(|| PromotedSurvival { - blocks: probe.blocks, - promoted_bytes: usize::try_from(probe.promoted_bytes).unwrap_or(usize::MAX), - live_bytes: usize::try_from(probe.live_bytes).unwrap_or(usize::MAX), - }); - #[cfg(test)] - LAST_SURVIVAL_FOR_TESTS.with(|c| { - c.set( - survival - .as_ref() - .map(|s| (s.blocks, s.promoted_bytes, s.live_bytes)), - ) - }); - survival -} - -#[cfg(test)] -thread_local! { - static LAST_SURVIVAL_FOR_TESTS: std::cell::Cell> = - const { std::cell::Cell::new(None) }; -} - -/// `(blocks, promoted bytes, live bytes)` of the last probe taken, when it -/// measured (tests only). -#[cfg(test)] -pub(super) fn last_survival_for_tests() -> Option<(usize, usize, usize)> { - LAST_SURVIVAL_FOR_TESTS.with(std::cell::Cell::get) -} - -/// Sabotage switch for the survival tests: the cohort full measures but does -/// not feed the promotion predictor. Test builds only. -#[cfg(test)] -pub(super) mod survival_sabotage { - use std::cell::Cell; - - thread_local! { - static UNFED: Cell = const { Cell::new(false) }; - } - - pub(in crate::gc) fn unfed() -> bool { - UNFED.with(Cell::get) - } - - pub(crate) struct Guard(bool); - - impl Guard { - pub(crate) fn arm() -> Self { - Self(UNFED.with(|s| s.replace(true))) - } - } - - impl Drop for Guard { - fn drop(&mut self) { - UNFED.with(|s| s.set(self.0)); - } - } -} - /// Sabotage switch for the cohort tests: every cohort full counts as /// productive, so the bound never backs off. Test builds only. #[cfg(test)] diff --git a/crates/perry-runtime/src/gc/promoted_cohort/survival.rs b/crates/perry-runtime/src/gc/promoted_cohort/survival.rs new file mode 100644 index 0000000000..bef4ed9cbd --- /dev/null +++ b/crates/perry-runtime/src/gc/promoted_cohort/survival.rs @@ -0,0 +1,423 @@ +//! What a promoted-cohort full's own mark says about the blocks the nursery +//! minor at the same safepoint promoted (#10241). +//! +//! # The defect this answers +//! +//! A minor promotes in place, and may skip its trace, on the strength of the +//! PREVIOUS minor's young-survival ratio. An untraced run re-measures only when +//! its byte budget runs out, and every full resets that budget +//! (`note_full_collection_reclaimed_old_gen`). A cohort full every ~`live` +//! promoted bytes therefore keeps a workload that turned from building a live +//! set to churning on the untraced path indefinitely: nothing measures the +//! churn, every minor promotes it, and every cohort full marks the whole live +//! set to reclaim it (`14_grow_then_churn`: 13 cohort fulls, 0 copied objects, +//! wall +38 %). +//! +//! # Why the full's survival alone is not the answer +//! +//! The full measures exactly: of the bytes the last minor promoted, how many +//! are still reachable. On the probe that is 4‰. But a parse loop measures +//! 250–500‰ there (`records_array_20m:parse` 500, `records_array_8m:scan` 250): +//! each tree's top-level array is born old, the tree's young records stay +//! reachable through that array's remembered slots, and so every minor promotes +//! the dead previous tree(s) with the live current one. Feeding that 500‰ to +//! the predictor makes the next minor evacuate — and the evacuating minor +//! copies both trees (~95 ms on 20m), because to a minor the dead tree is live: +//! its parent is old, and a minor treats every old object as live. +//! +//! The predictor describes what a MINOR measures, so the full's figure may +//! replace it only when the two provably coincide. They differ by exactly the +//! young bytes a minor reaches through a remembered parent the full found dead. +//! +//! # The discriminator +//! +//! The minor notes its remembered set as its own dirty scan sees it (the dirty +//! old pages; `note_minor_remembered_parents`). After the full's mark and before +//! its sweep, every object on those pages that the mark left unmarked has its +//! slots on those pages read, exactly the slots the minor's dirty scan visits: +//! if one refers into the blocks the minor promoted, a minor would have found +//! young bytes live that the full found dead, and the measurement is not the +//! minor's (`MinorView::DeadParent`). If none does, a minor's reachability over +//! that young generation equals the full's — same safepoint, same roots, and +//! every old parent that could root a young object was itself proven live — so +//! the survival is exactly the ratio a traced minor would have measured +//! (`MinorView::Exact`). Parents the remembered set names by header rather than +//! by page (external slot pages, the fallback set) are not re-derived, so any +//! such entry makes the view `Unverifiable`. +//! +//! On a parse loop the dead previous tree's top array is such a parent — and +//! its dirty pages are exactly the slots written since the previous minor, so +//! the first slot read hits. On the probe the only remembered parent is the +//! 32-slot ring, which is live. +//! +//! # Exact or nothing +//! +//! The blocks are the ones the minor's promotion walk recorded for census +//! adoption (`trace::adopt_census`), keyed by data address with the bump extent +//! and header bytes they held at promotion. Each is accounted when the full's +//! sweep either walks it whole at the same extent (its live bytes are the +//! sweep's own `arena_live_bytes` delta) or reclaims it unwalked as a dead block +//! (live 0). A block the sweep reaches any other way, at another extent, or not +//! at all leaves the measurement unset. + +use super::super::*; +use std::cell::RefCell; + +/// Whether the full's survival is the survival a minor would have measured. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::gc) enum MinorView { + /// No sweep-start check ran. + Unchecked, + /// No remembered parent of the minor that the full found dead refers into + /// the promoted blocks. + Exact, + /// A dead remembered parent refers into the promoted blocks. + DeadParent, + /// The minor's remembered set was not noted, or names parents by header. + Unverifiable, +} + +impl MinorView { + pub(in crate::gc) fn as_str(self) -> &'static str { + match self { + Self::Unchecked => "unchecked", + Self::Exact => "exact", + Self::DeadParent => "dead_parent", + Self::Unverifiable => "unverifiable", + } + } +} + +/// The measurement a probe produced. +pub(in crate::gc) struct PromotedSurvival { + pub(in crate::gc) blocks: usize, + pub(in crate::gc) promoted_bytes: usize, + pub(in crate::gc) live_bytes: usize, + pub(in crate::gc) minor_view: MinorView, +} + +impl PromotedSurvival { + pub(in crate::gc) fn permille(&self) -> Option { + survival_permille(self.promoted_bytes, self.live_bytes) + } +} + +pub(in crate::gc) fn survival_permille(promoted_bytes: usize, live_bytes: usize) -> Option { + (promoted_bytes > 0).then(|| { + (live_bytes as u64) + .saturating_mul(1000) + .checked_div(promoted_bytes as u64) + .unwrap_or(0) + .min(1000) + }) +} + +/// The minor's remembered set, as page keys: nothing here is an address the +/// full dereferences. +pub(in crate::gc) struct RememberedParents { + dirty_old_pages: crate::fast_hash::PtrHashSet, + dirty_pages: crate::fast_hash::PtrHashSet, + header_parents: usize, +} + +struct SurvivalProbe { + /// Recorded blocks not yet accounted: data address -> (extent, bytes). + pending: crate::fast_hash::PtrHashMap, + /// `(data, data + extent)` of every recorded block, sorted. + ranges: Vec<(usize, usize)>, + parents: Option, + minor_view: MinorView, + blocks: usize, + promoted_bytes: u64, + live_bytes: u64, + exact: bool, +} + +crate::perry_thread_local! { + /// The remembered set of the minor recording for this safepoint's cohort + /// full. Page keys and a count only. + static MINOR_REMEMBERED_PARENTS: RefCell> = + const { RefCell::new(None) }; + /// The armed probe of the cohort full in progress. Block data addresses + /// are identity keys and range bounds for pointer comparisons; nothing is + /// read through them. + static SURVIVAL_PROBE: RefCell> = const { RefCell::new(None) }; +} + +/// A copying minor took its remembered-set snapshot. Kept only while the +/// promotion census is recording for a cohort full at this safepoint. +pub(in crate::gc) fn note_minor_remembered_parents(snapshot: &RememberedDirtySnapshot) { + if !super::super::trace::adopt_census::recording() { + return; + } + let parents = RememberedParents { + dirty_old_pages: snapshot.dirty_old_pages.clone(), + dirty_pages: snapshot.dirty_pages.clone(), + header_parents: snapshot.external_dirty_entries.len() + snapshot.fallback_headers.len(), + }; + MINOR_REMEMBERED_PARENTS.with(|p| *p.borrow_mut() = Some(parents)); +} + +pub(in crate::gc) fn clear_minor_remembered_parents() { + MINOR_REMEMBERED_PARENTS.with(|p| *p.borrow_mut() = None); +} + +/// Arm the probe over the recorded `(data, extent, bytes)` blocks before the +/// cohort full, with the remembered set the same minor noted. +pub(in crate::gc) fn arm_survival_probe(blocks: Vec<(usize, usize, u64)>) { + let parents = MINOR_REMEMBERED_PARENTS.with(|p| p.borrow_mut().take()); + if blocks.is_empty() { + SURVIVAL_PROBE.with(|p| *p.borrow_mut() = None); + return; + } + let mut pending = crate::fast_hash::new_ptr_hash_map(); + let mut ranges = Vec::with_capacity(blocks.len()); + let mut promoted_bytes = 0u64; + for (data, extent, bytes) in blocks { + promoted_bytes = promoted_bytes.saturating_add(bytes); + pending.insert(data, (extent, bytes)); + ranges.push((data, data.saturating_add(extent))); + } + ranges.sort_unstable(); + let minor_view = match &parents { + Some(parents) if parents.header_parents == 0 => MinorView::Unchecked, + _ => MinorView::Unverifiable, + }; + let probe = SurvivalProbe { + blocks: pending.len(), + pending, + ranges, + parents, + minor_view, + promoted_bytes, + live_bytes: 0, + exact: true, + }; + SURVIVAL_PROBE.with(|p| *p.borrow_mut() = Some(probe)); +} + +/// Read once per synchronous full sweep. +pub(in crate::gc) fn survival_probe_armed() -> bool { + SURVIVAL_PROBE.with(|p| p.borrow().is_some()) +} + +/// A synchronous full's marks are final and nothing is swept: decide whether a +/// remembered parent of the minor that the mark left unmarked refers into the +/// promoted blocks. +pub(in crate::gc) fn check_minor_view_at_full_sweep_start() { + SURVIVAL_PROBE.with(|p| { + let mut probe = p.borrow_mut(); + let Some(probe) = probe.as_mut() else { + return; + }; + if probe.minor_view != MinorView::Unchecked { + return; + } + let Some(parents) = probe.parents.take() else { + probe.minor_view = MinorView::Unverifiable; + return; + }; + #[cfg(test)] + if sabotage::skipping_parent_check() { + probe.minor_view = MinorView::Exact; + return; + } + let ranges = &probe.ranges; + let mut dead_parent = false; + crate::arena::old_arena_walk_objects_on_pages(&parents.dirty_old_pages, |header| { + if !dead_parent { + // SAFETY: the old page index names headers of live arena + // allocations; the full has not swept anything yet. + dead_parent = unsafe { + dead_parent_refers_into(header as *mut GcHeader, &parents.dirty_pages, ranges) + }; + } + }); + probe.minor_view = if dead_parent { + MinorView::DeadParent + } else { + MinorView::Exact + }; + }); +} + +/// Does `header`, unmarked by the full, hold on a dirty page a slot that +/// refers into `ranges`? Visits exactly the slots the minor's dirty scan +/// (`scan_dirty_object_slots`) visits, without its accounting. +/// +/// # Safety +/// `header` is an arena header from the old page index, before the sweep. +unsafe fn dead_parent_refers_into( + header: *mut GcHeader, + dirty_pages: &crate::fast_hash::PtrHashSet, + ranges: &[(usize, usize)], +) -> bool { + if !plausible_gc_header(header, true) || (*header).gc_flags & GC_FLAG_MARKED != 0 { + return false; + } + let user = (header as *mut u8).add(GC_HEADER_SIZE) as usize; + if !matches!( + crate::arena::classify_heap_generation(user), + crate::arena::HeapGeneration::Old + ) { + return false; + } + let mut hit = false; + let mut scratch = RememberedSetTraceStats::default(); + visit_gc_rewrite_slot_descriptors(header, |descriptor| unsafe { + if hit { + return; + } + match descriptor { + GcMutableSlotDescriptor::Slot(slot) => { + if !crate::weakref::is_weak_target_trace_slot(header, slot.slot) + && dirty_pages_contains_addr(dirty_pages, slot.slot as usize) + { + hit = bits_refer_into(*slot.slot, ranges); + } + } + GcMutableSlotDescriptor::Range { range, .. } => { + let weak = crate::weakref::header_may_hold_weak_target_slots(header); + // Newest slots first: the ones written since the last minor. + for (start, end) in dirty_slot_ranges_for(range, dirty_pages, &mut scratch) + .into_iter() + .rev() + { + for i in (start..end).rev() { + let slot = range.slot(i); + if weak && crate::weakref::is_weak_target_trace_slot(header, slot) { + continue; + } + if bits_refer_into(*slot, ranges) { + hit = true; + return; + } + } + } + } + GcMutableSlotDescriptor::PointerFreeRange(_) => {} + } + }); + hit +} + +/// Does a slot's value (NaN-boxed or a raw address) point into `ranges`? +fn bits_refer_into(bits: u64, ranges: &[(usize, usize)]) -> bool { + let addr = match bits >> 48 { + 0x7FFA | 0x7FFD | 0x7FFF => (bits & 0x0000_FFFF_FFFF_FFFF) as usize, + 0 => bits as usize, + _ => return false, + }; + if addr == 0 { + return false; + } + let idx = ranges.partition_point(|&(start, _)| start <= addr); + idx > 0 && addr < ranges[idx - 1].1 +} + +/// The sweep walked the block at `data` whole, to `extent`, and kept `live` +/// bytes of it. +pub(in crate::gc) fn note_probe_block_swept(data: usize, extent: usize, live: u64) { + account_probe_block(data, extent, live); +} + +/// The sweep reclaimed the block at `data` without walking it: nothing in it +/// was reached. +pub(in crate::gc) fn note_probe_block_skipped(data: usize, extent: usize) { + account_probe_block(data, extent, 0); +} + +fn account_probe_block(data: usize, extent: usize, live: u64) { + SURVIVAL_PROBE.with(|p| { + let mut probe = p.borrow_mut(); + let Some(probe) = probe.as_mut() else { + return; + }; + let Some((recorded_extent, _)) = probe.pending.remove(&data) else { + return; + }; + if recorded_extent != extent { + probe.exact = false; + } + probe.live_bytes = probe.live_bytes.saturating_add(live); + }); +} + +/// Disarm the probe; the measurement when every recorded block was accounted. +pub(in crate::gc) fn take_survival_probe() -> Option { + let probe = SURVIVAL_PROBE.with(|p| p.borrow_mut().take())?; + let survival = (probe.exact && probe.pending.is_empty()).then(|| PromotedSurvival { + blocks: probe.blocks, + promoted_bytes: usize::try_from(probe.promoted_bytes).unwrap_or(usize::MAX), + live_bytes: usize::try_from(probe.live_bytes).unwrap_or(usize::MAX), + minor_view: probe.minor_view, + }); + #[cfg(test)] + LAST_SURVIVAL_FOR_TESTS.with(|c| { + *c.borrow_mut() = survival + .as_ref() + .map(|s| (s.blocks, s.promoted_bytes, s.live_bytes, s.minor_view)); + }); + survival +} + +#[cfg(test)] +thread_local! { + static LAST_SURVIVAL_FOR_TESTS: RefCell> = + const { RefCell::new(None) }; +} + +/// `(blocks, promoted bytes, live bytes, minor view)` of the last probe taken, +/// when it measured (tests only). +#[cfg(test)] +pub(in crate::gc) fn last_survival_for_tests() -> Option<(usize, usize, usize, MinorView)> { + LAST_SURVIVAL_FOR_TESTS.with(|c| *c.borrow()) +} + +/// Sabotage switches for the survival tests. Test builds only. +#[cfg(test)] +pub(in crate::gc) mod sabotage { + use std::cell::Cell; + + thread_local! { + static UNFED: Cell = const { Cell::new(false) }; + static SKIP_PARENT_CHECK: Cell = const { Cell::new(false) }; + } + + /// The cohort full measures but does not feed the predictor. + pub(in crate::gc) fn unfed() -> bool { + UNFED.with(Cell::get) + } + + /// Every measurement counts as the minor's view. + pub(super) fn skipping_parent_check() -> bool { + SKIP_PARENT_CHECK.with(Cell::get) + } + + pub(in crate::gc) struct Guard { + flag: &'static std::thread::LocalKey>, + previous: bool, + } + + impl Guard { + pub(in crate::gc) fn unfed() -> Self { + Self { + flag: &UNFED, + previous: UNFED.with(|s| s.replace(true)), + } + } + + pub(in crate::gc) fn skip_parent_check() -> Self { + Self { + flag: &SKIP_PARENT_CHECK, + previous: SKIP_PARENT_CHECK.with(|s| s.replace(true)), + } + } + } + + impl Drop for Guard { + fn drop(&mut self) { + self.flag.with(|s| s.set(self.previous)); + } + } +} diff --git a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs index 9e933308bf..c2fbf17c12 100644 --- a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs @@ -16,6 +16,7 @@ use super::super::policy::{ run_promoted_cohort_full_if_due, GC_LAST_OLD_RECLAIM_IN_USE_BYTES, GC_MAJOR_PACING_RETAINING, }; use super::super::promoted_cohort as cohort; +use super::super::promoted_cohort::survival; use super::super::*; use super::support::*; @@ -193,8 +194,8 @@ fn below_the_bound_no_cohort_full_runs_and_the_dead_promoted_object_stays() { /// Outcome of one untraced promotion, a cohort full at the same safepoint, and /// the minor after it. struct SurvivalOutcome { - /// `(blocks, promoted bytes, live bytes)` the full measured. - measured: Option<(usize, usize, usize)>, + /// `(blocks, promoted bytes, live bytes, minor view)` the full measured. + measured: Option<(usize, usize, usize, survival::MinorView)>, predictor_after_full: Option, next_minor_in_place: bool, next_minor_untraced: bool, @@ -248,12 +249,12 @@ fn promote_then_cohort_full_then_minor(live: bool, fed: bool) -> SurvivalOutcome } cohort::seed_for_tests(cohort::bound_bytes(), 0, 0); let ran = { - let _sabotage = (!fed).then(cohort::survival_sabotage::Guard::arm); + let _sabotage = (!fed).then(survival::sabotage::Guard::unfed); run_promoted_cohort_full_if_due() }; adopt_census::discard(); assert!(ran, "premise: the cohort full ran"); - let measured = cohort::last_survival_for_tests(); + let measured = survival::last_survival_for_tests(); let predictor_after_full = super::super::last_young_survival_permille(); rooted_young_strings(1, COUNT); @@ -271,7 +272,7 @@ fn promote_then_cohort_full_then_minor(live: bool, fed: bool) -> SurvivalOutcome outcome } -fn permille((_, promoted, live): (usize, usize, usize)) -> u64 { +fn permille((_, promoted, live, _): (usize, usize, usize, survival::MinorView)) -> u64 { (live as u64 * 1000 / promoted as u64).min(1000) } diff --git a/crates/perry-runtime/src/gc/trace/adopt_census.rs b/crates/perry-runtime/src/gc/trace/adopt_census.rs index d0005c9758..1d01bb97c2 100644 --- a/crates/perry-runtime/src/gc/trace/adopt_census.rs +++ b/crates/perry-runtime/src/gc/trace/adopt_census.rs @@ -73,9 +73,12 @@ crate::perry_thread_local! { /// Record the census facts of the blocks the next promotion walk promotes. pub(crate) fn begin_recording() { STATE.with(|s| *s.borrow_mut() = State::Recording(crate::fast_hash::new_ptr_hash_map())); + // #10241: the minor about to run notes its own remembered set; nothing an + // earlier safepoint noted may reach this one's cohort full. + super::super::promoted_cohort::survival::clear_minor_remembered_parents(); } -fn recording() -> bool { +pub(crate) fn recording() -> bool { STATE.with(|s| matches!(*s.borrow(), State::Recording(_))) } @@ -104,6 +107,7 @@ pub(crate) fn begin_adopting() { /// Drop every record: the safepoint is returning to the mutator. pub(crate) fn discard() { STATE.with(|s| *s.borrow_mut() = State::Off); + super::super::promoted_cohort::survival::clear_minor_remembered_parents(); } /// `(data, extent, bytes)` of every block the minor at this safepoint recorded From 577b2f07377b9a68ed2ca71ebc1ef941da6d60db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 07:53:03 +0200 Subject: [PATCH 33/40] gc: test the dead-remembered-parent gate on the cohort survival feed (#10182) --- .../src/gc/tests/promoted_cohort.rs | 166 +++++++++++++++--- 1 file changed, 138 insertions(+), 28 deletions(-) diff --git a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs index c2fbf17c12..eaa451ef16 100644 --- a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs @@ -7,9 +7,11 @@ //! follows it reclaims a promoted object that died. //! //! #10241: the cohort full measures the survival of what the minor at its own -//! safepoint promoted, and a dead same-safepoint cohort turns the next minor -//! back into an evacuating one (with a sabotaged twin that does not feed the -//! predictor, and a live cohort that leaves it alone). +//! safepoint promoted. A dead same-safepoint cohort turns the next minor back +//! into an evacuating one (sabotaged twin: the predictor is not fed), a live +//! one leaves the predictor alone, and a dead remembered parent referring into +//! the cohort — the parse-loop shape, where a minor would have measured the +//! cohort live — feeds nothing (sabotaged twin: the parent check is skipped). use super::super::policy::{ credit_promoted_bytes_to_old_baseline, old_reclaim_pressure_due, @@ -191,6 +193,27 @@ fn below_the_bound_no_cohort_full_runs_and_the_dead_promoted_object_stays() { ); } +/// What the young population promoted at the cohort's safepoint looks like at +/// the full. +#[derive(Clone, Copy, PartialEq)] +enum CohortShape { + /// Still rooted. + Live, + /// Dropped; a rooted old parent keeps one young string on a dirty page, so + /// the full checks a remembered parent and finds it live. + DeadWithLiveParent, + /// Dropped; an unrooted old parent keeps young strings on a dirty page — + /// the born-old array of a parsed tree that died. + DeadWithDeadParent, +} + +#[derive(Clone, Copy, PartialEq)] +enum Sabotage { + None, + Unfed, + SkipParentCheck, +} + /// Outcome of one untraced promotion, a cohort full at the same safepoint, and /// the minor after it. struct SurvivalOutcome { @@ -213,12 +236,24 @@ fn rooted_young_strings(slot: u32, count: usize) -> usize { first } +/// An old object whose `fields` young strings are stored through the runtime +/// barrier, so its page is in the remembered set the next minor snapshots. +fn old_parent_of_young_strings(fields: u32) -> usize { + let (parent, slots) = unsafe { alloc_old_test_object(fields) }; + for i in 0..fields as usize { + let bits = string_bits(young_leaf()); + unsafe { + *slots.add(i) = bits; + } + runtime_write_barrier_slot(parent as usize, unsafe { slots.add(i) } as usize, bits); + } + parent as usize +} + /// A young population promoted whole and untraced by a minor that records its -/// blocks, then the cohort full at the same safepoint — with that population -/// still rooted (`live`) or dropped — then a minor over a fresh rooted -/// population. `fed == false` arms the sabotage that keeps the full's -/// measurement away from the predictor. -fn promote_then_cohort_full_then_minor(live: bool, fed: bool) -> SurvivalOutcome { +/// blocks, then the cohort full at the same safepoint in `shape`, then a minor +/// over a fresh rooted population. +fn promote_then_cohort_full_then_minor(shape: CohortShape, sabotage: Sabotage) -> SurvivalOutcome { use super::super::trace::adopt_census; let _guard = CopyingNurseryTestGuard::new(4); let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); @@ -231,6 +266,21 @@ fn promote_then_cohort_full_then_minor(live: bool, fed: bool) -> SurvivalOutcome crate::arena::pointer_in_nursery(probe_leaf), "premise: young population" ); + match shape { + CohortShape::Live => {} + CohortShape::DeadWithLiveParent => { + js_shadow_slot_set(2, ptr_bits(old_parent_of_young_strings(1))); + } + CohortShape::DeadWithDeadParent => { + old_parent_of_young_strings(64); + } + } + if shape != CohortShape::Live { + assert!( + super::super::barrier::remembered_dirty_page_count() > 0, + "premise: the old parent's page is remembered" + ); + } let untraced_before = untraced_promotion_cycles(); adopt_census::begin_recording(); let trace = collect_minor_trace(GcTriggerKind::Direct); @@ -244,12 +294,16 @@ fn promote_then_cohort_full_then_minor(live: bool, fed: bool) -> SurvivalOutcome 1, "premise: the promotion skipped the trace" ); - if !live { + if shape != CohortShape::Live { js_shadow_slot_set(0, crate::value::TAG_UNDEFINED); } cohort::seed_for_tests(cohort::bound_bytes(), 0, 0); let ran = { - let _sabotage = (!fed).then(survival::sabotage::Guard::unfed); + let _sabotage = match sabotage { + Sabotage::None => None, + Sabotage::Unfed => Some(survival::sabotage::Guard::unfed()), + Sabotage::SkipParentCheck => Some(survival::sabotage::Guard::skip_parent_check()), + }; run_promoted_cohort_full_if_due() }; adopt_census::discard(); @@ -266,8 +320,9 @@ fn promote_then_cohort_full_then_minor(live: bool, fed: bool) -> SurvivalOutcome next_minor_in_place: next.copying_nursery.in_place_promotion, next_minor_untraced: untraced_promotion_cycles() - untraced_before == 1, }; - js_shadow_slot_set(0, crate::value::TAG_UNDEFINED); - js_shadow_slot_set(1, crate::value::TAG_UNDEFINED); + for slot in 0..3 { + js_shadow_slot_set(slot, crate::value::TAG_UNDEFINED); + } cohort::seed_for_tests(0, 0, 0); outcome } @@ -276,9 +331,9 @@ fn permille((_, promoted, live, _): (usize, usize, usize, survival::MinorView)) (live as u64 * 1000 / promoted as u64).min(1000) } -#[test] -fn a_dead_same_safepoint_cohort_turns_the_next_minor_into_an_evacuation() { - let outcome = promote_then_cohort_full_then_minor(false, true); +fn dead_cohort_measurement( + outcome: &SurvivalOutcome, +) -> (usize, usize, usize, survival::MinorView) { let measured = outcome .measured .expect("the full's sweep accounted every recorded block"); @@ -286,14 +341,26 @@ fn a_dead_same_safepoint_cohort_turns_the_next_minor_into_an_evacuation() { measured.0 >= 1 && measured.1 > 0, "the probe measured the promoted blocks: {measured:?}" ); - let survival = permille(measured); assert!( - survival < super::super::PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, - "the dropped cohort is dead by the full: {survival} permille ({measured:?})" + permille(measured) < super::super::PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, + "the dropped cohort is dead by the full: {measured:?}" + ); + measured +} + +#[test] +fn a_dead_same_safepoint_cohort_turns_the_next_minor_into_an_evacuation() { + let outcome = + promote_then_cohort_full_then_minor(CohortShape::DeadWithLiveParent, Sabotage::None); + let measured = dead_cohort_measurement(&outcome); + assert_eq!( + measured.3, + survival::MinorView::Exact, + "the only remembered parent is live, so a minor measures what the full did" ); assert_eq!( outcome.predictor_after_full, - Some(survival), + Some(permille(measured)), "the full's measurement replaces the ratio the promotion was admitted on" ); assert!( @@ -304,11 +371,13 @@ fn a_dead_same_safepoint_cohort_turns_the_next_minor_into_an_evacuation() { #[test] fn sabotaged_unfed_predictor_promotes_the_next_minor_untraced() { - let outcome = promote_then_cohort_full_then_minor(false, false); - let measured = outcome.measured.expect("premise: the probe still measured"); - assert!( - permille(measured) < super::super::PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, - "premise: the cohort is dead by the full ({measured:?})" + let outcome = + promote_then_cohort_full_then_minor(CohortShape::DeadWithLiveParent, Sabotage::Unfed); + let measured = dead_cohort_measurement(&outcome); + assert_eq!( + measured.3, + survival::MinorView::Exact, + "premise: exact view" ); assert_eq!( outcome.predictor_after_full, @@ -324,14 +393,13 @@ fn sabotaged_unfed_predictor_promotes_the_next_minor_untraced() { #[test] fn a_live_same_safepoint_cohort_leaves_the_predictor_at_retained() { - let outcome = promote_then_cohort_full_then_minor(true, true); + let outcome = promote_then_cohort_full_then_minor(CohortShape::Live, Sabotage::None); let measured = outcome .measured .expect("the full's sweep accounted every recorded block"); - let survival = permille(measured); assert!( - survival >= super::super::PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, - "the rooted cohort survives the full: {survival} permille ({measured:?})" + permille(measured) >= super::super::PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, + "the rooted cohort survives the full: {measured:?}" ); assert_eq!( outcome.predictor_after_full, @@ -343,3 +411,45 @@ fn a_live_same_safepoint_cohort_leaves_the_predictor_at_retained() { "a retained cohort keeps the next minor on the untraced promotion" ); } + +#[test] +fn a_dead_remembered_parent_keeps_a_dead_cohort_from_feeding_the_predictor() { + let outcome = + promote_then_cohort_full_then_minor(CohortShape::DeadWithDeadParent, Sabotage::None); + let measured = dead_cohort_measurement(&outcome); + assert_eq!( + measured.3, + survival::MinorView::DeadParent, + "the dead old parent refers into the promoted blocks: a minor would have \ + measured them live" + ); + assert_eq!( + outcome.predictor_after_full, + Some(1000), + "a survival no minor would measure is not fed" + ); + assert!( + outcome.next_minor_in_place && outcome.next_minor_untraced, + "the next minor keeps the untraced promotion (records_array_20m:parse)" + ); +} + +#[test] +fn sabotaged_parent_check_feeds_a_parse_loop_cohort_into_an_evacuation() { + let outcome = promote_then_cohort_full_then_minor( + CohortShape::DeadWithDeadParent, + Sabotage::SkipParentCheck, + ); + let measured = dead_cohort_measurement(&outcome); + assert_eq!( + measured.3, + survival::MinorView::Exact, + "premise: check skipped" + ); + assert!( + !outcome.next_minor_in_place && !outcome.next_minor_untraced, + "without the parent check the dead-parent cohort is fed and the next \ + minor evacuates — the traced minor that cost records_array_20m:parse \ + ~95 ms per cohort full" + ); +} From 99dfcb82c046df619765244c39a0c85eebc45b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 07:54:38 +0200 Subject: [PATCH 34/40] gc: classify the cohort survival probe's thread-locals and re-pin the census window (#10182) --- scripts/gc_runtime_root_holders.json | 36 +++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index ddf62cadff..52345c5ddb 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -313,7 +313,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes — in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes — in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -329,9 +329,9 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "5c151725460ffb92a55a6bee781123ef5159263b4ce5958d16570f78216e0d67", - "crates/perry-runtime/src/gc/cycle.rs": "9ebee6df0040a50a7f24bd098ce042512a93a1a7ac232346b95fe66cfdf9c0fd", + "crates/perry-runtime/src/gc/cycle.rs": "b035dcb44df029358cbab0afaa526e8e506765f5178034663257e18ceefaf9df", "crates/perry-runtime/src/gc/mod.rs": "9fedd2790f48154aaeceefb4805d3fbaa2fdf3c407529b326425fde86c2bf9a5", - "crates/perry-runtime/src/gc/policy.rs": "d588a0a3ffe21f8523d419fdced0942ca36b63c4d6c38faac93db1042304f8a9", + "crates/perry-runtime/src/gc/policy.rs": "853e9bf44a03d3d14a335c5c1566732d449bbb2b62da3d656f5578f8c4805ccc", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } } @@ -444,6 +444,36 @@ "verdict": "not_a_gc_pointer", "why": "#10182: a `Cell` byte count, the bytes promoted into old-gen since the last full collection, read by the promoted-cohort bound. Holds no address." }, + { + "file": "crates/perry-runtime/src/gc/promoted_cohort/survival.rs", + "name": "LAST_SURVIVAL_FOR_TESTS", + "verdict": "test_only", + "why": "Declared under #[cfg(test)]; a `RefCell>` holding the block count, promoted and live byte counts and view verdict of the last #10241 survival probe, read by the promoted-cohort tests. Byte counts only; absent from shipped binaries." + }, + { + "file": "crates/perry-runtime/src/gc/promoted_cohort/survival.rs", + "name": "MINOR_REMEMBERED_PARENTS", + "verdict": "not_a_gc_pointer", + "why": "#10241: the remembered set of the copying minor recording for a cohort full at the same safepoint, as generation PAGE KEYS (`addr >> 12`, two `PtrHashSet`) plus a count of header-named entries. Page keys are not addresses of any object and are never dereferenced; the cohort full re-derives the objects on those pages from the old page index after its own mark. Cleared by `adopt_census::begin_recording`/`discard` and consumed by `arm_survival_probe`, so it never outlives the safepoint." + }, + { + "file": "crates/perry-runtime/src/gc/promoted_cohort/survival.rs", + "name": "SKIP_PARENT_CHECK", + "verdict": "test_only", + "why": "Declared under #[cfg(test)] in the survival sabotage module; a `Cell` switch. Absent from shipped binaries." + }, + { + "file": "crates/perry-runtime/src/gc/promoted_cohort/survival.rs", + "name": "SURVIVAL_PROBE", + "verdict": "not_a_gc_pointer", + "why": "#10241: the promoted-cohort full's survival probe. Holds arena BLOCK data addresses (block bases, not object pointers) as hash keys and `(start, end)` range bounds, byte counts and a view enum, plus the page keys taken from MINOR_REMEMBERED_PARENTS. Nothing is read through the block addresses: the sweep compares them with its own block snapshot and the parent check compares slot values against the ranges. Armed immediately before a synchronous non-moving full and taken immediately after it, at one safepoint with no mutator code in between." + }, + { + "file": "crates/perry-runtime/src/gc/promoted_cohort/survival.rs", + "name": "UNFED", + "verdict": "test_only", + "why": "Declared under #[cfg(test)] in the survival sabotage module; a `Cell` switch. Absent from shipped binaries." + }, { "file": "crates/perry-runtime/src/gc/survival_diag.rs", "name": "MINOR_SEQ", From 6a44e3274770278548457dbf605af91d6b85146f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 08:04:18 +0200 Subject: [PATCH 35/40] gc: leave the remembered-parents record untouched on safepoints that did not record (#10182) --- .../src/gc/trace/adopt_census.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/gc/trace/adopt_census.rs b/crates/perry-runtime/src/gc/trace/adopt_census.rs index 1d01bb97c2..c8996f6e9f 100644 --- a/crates/perry-runtime/src/gc/trace/adopt_census.rs +++ b/crates/perry-runtime/src/gc/trace/adopt_census.rs @@ -84,13 +84,17 @@ pub(crate) fn recording() -> bool { /// The minor is over: keep what it recorded for a full at this safepoint. pub(crate) fn finish_recording() { - STATE.with(|s| { + let ready = STATE.with(|s| { let mut state = s.borrow_mut(); *state = match std::mem::replace(&mut *state, State::Off) { State::Recording(map) if !map.is_empty() => State::Ready(map), _ => State::Off, }; + matches!(*state, State::Ready(_)) }); + if !ready { + super::super::promoted_cohort::survival::clear_minor_remembered_parents(); + } } /// The full about to start may adopt the records. @@ -106,8 +110,17 @@ pub(crate) fn begin_adopting() { /// Drop every record: the safepoint is returning to the mutator. pub(crate) fn discard() { - STATE.with(|s| *s.borrow_mut() = State::Off); - super::super::promoted_cohort::survival::clear_minor_remembered_parents(); + let was_off = STATE.with(|s| { + matches!( + std::mem::replace(&mut *s.borrow_mut(), State::Off), + State::Off + ) + }); + // Every nursery safepoint discards; only one that recorded can have noted + // a remembered set, so the common path leaves that thread-local untouched. + if !was_off { + super::super::promoted_cohort::survival::clear_minor_remembered_parents(); + } } /// `(data, extent, bytes)` of every block the minor at this safepoint recorded From 19adcd89824b1f5f105169c9cf943cd41cfff44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 08:14:29 +0200 Subject: [PATCH 36/40] gc: note the remembered parents only for a promoting minor, as one page set (#10182) --- crates/perry-runtime/src/gc/copying.rs | 7 ++-- .../src/gc/promoted_cohort/survival.rs | 32 ++++++++++--------- scripts/gc_runtime_root_holders.json | 2 +- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 8ff4869d9a..ff89e2e374 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -1354,8 +1354,11 @@ pub(super) fn run_copied_minor_attempt( let remembered_phase_start = PhaseDiag::start(&phase_diag); let snapshot = remembered_dirty_snapshot(); // #10241: a cohort full at this safepoint asks whether a dead parent in - // this remembered set held what this minor promotes. - super::promoted_cohort::survival::note_minor_remembered_parents(&snapshot); + // this remembered set held what this minor promotes. Only a promoting + // minor records blocks for it to ask about. + if promoting_in_place { + super::promoted_cohort::survival::note_minor_remembered_parents(&snapshot); + } // #9754: objects whose every slot the dirty scan visited in-body — the // post-cycle coverage restore skips them (see `scan_dirty_object_slots`). // #9835: this set is rebuilt from EMPTY on every minor and reaches ~1,000 diff --git a/crates/perry-runtime/src/gc/promoted_cohort/survival.rs b/crates/perry-runtime/src/gc/promoted_cohort/survival.rs index bef4ed9cbd..35f2657aff 100644 --- a/crates/perry-runtime/src/gc/promoted_cohort/survival.rs +++ b/crates/perry-runtime/src/gc/promoted_cohort/survival.rs @@ -113,11 +113,10 @@ pub(in crate::gc) fn survival_permille(promoted_bytes: usize, live_bytes: usize) } /// The minor's remembered set, as page keys: nothing here is an address the -/// full dereferences. +/// full dereferences. Kept only when the remembered set names no parent by +/// header, so its dirty pages are exactly its dirty old pages. pub(in crate::gc) struct RememberedParents { dirty_old_pages: crate::fast_hash::PtrHashSet, - dirty_pages: crate::fast_hash::PtrHashSet, - header_parents: usize, } struct SurvivalProbe { @@ -150,12 +149,14 @@ pub(in crate::gc) fn note_minor_remembered_parents(snapshot: &RememberedDirtySna if !super::super::trace::adopt_census::recording() { return; } - let parents = RememberedParents { + // Parents named by header are not re-derivable from page keys: leave the + // record unset, and the view `Unverifiable`. + let parents = (snapshot.external_dirty_entries.is_empty() + && snapshot.fallback_headers.is_empty()) + .then(|| RememberedParents { dirty_old_pages: snapshot.dirty_old_pages.clone(), - dirty_pages: snapshot.dirty_pages.clone(), - header_parents: snapshot.external_dirty_entries.len() + snapshot.fallback_headers.len(), - }; - MINOR_REMEMBERED_PARENTS.with(|p| *p.borrow_mut() = Some(parents)); + }); + MINOR_REMEMBERED_PARENTS.with(|p| *p.borrow_mut() = parents); } pub(in crate::gc) fn clear_minor_remembered_parents() { @@ -179,9 +180,10 @@ pub(in crate::gc) fn arm_survival_probe(blocks: Vec<(usize, usize, u64)>) { ranges.push((data, data.saturating_add(extent))); } ranges.sort_unstable(); - let minor_view = match &parents { - Some(parents) if parents.header_parents == 0 => MinorView::Unchecked, - _ => MinorView::Unverifiable, + let minor_view = if parents.is_some() { + MinorView::Unchecked + } else { + MinorView::Unverifiable }; let probe = SurvivalProbe { blocks: pending.len(), @@ -224,13 +226,13 @@ pub(in crate::gc) fn check_minor_view_at_full_sweep_start() { } let ranges = &probe.ranges; let mut dead_parent = false; - crate::arena::old_arena_walk_objects_on_pages(&parents.dirty_old_pages, |header| { + let pages = &parents.dirty_old_pages; + crate::arena::old_arena_walk_objects_on_pages(pages, |header| { if !dead_parent { // SAFETY: the old page index names headers of live arena // allocations; the full has not swept anything yet. - dead_parent = unsafe { - dead_parent_refers_into(header as *mut GcHeader, &parents.dirty_pages, ranges) - }; + dead_parent = + unsafe { dead_parent_refers_into(header as *mut GcHeader, pages, ranges) }; } }); probe.minor_view = if dead_parent { diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 52345c5ddb..976884192d 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -454,7 +454,7 @@ "file": "crates/perry-runtime/src/gc/promoted_cohort/survival.rs", "name": "MINOR_REMEMBERED_PARENTS", "verdict": "not_a_gc_pointer", - "why": "#10241: the remembered set of the copying minor recording for a cohort full at the same safepoint, as generation PAGE KEYS (`addr >> 12`, two `PtrHashSet`) plus a count of header-named entries. Page keys are not addresses of any object and are never dereferenced; the cohort full re-derives the objects on those pages from the old page index after its own mark. Cleared by `adopt_census::begin_recording`/`discard` and consumed by `arm_survival_probe`, so it never outlives the safepoint." + "why": "#10241: the remembered set of the copying minor recording for a cohort full at the same safepoint, as generation PAGE KEYS (`addr >> 12`, one `PtrHashSet` of dirty old pages), kept only for a minor that promotes in place and whose remembered set names no parent by header. Page keys are not addresses of any object and are never dereferenced; the cohort full re-derives the objects on those pages from the old page index after its own mark. Cleared by `adopt_census::begin_recording`/`discard` and consumed by `arm_survival_probe`, so it never outlives the safepoint." }, { "file": "crates/perry-runtime/src/gc/promoted_cohort/survival.rs", From a6053c9faa4156d4b24615546a81c07beed104f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 08:56:21 +0200 Subject: [PATCH 37/40] gc: the promoted cohort counts in-place promotions only (#10182) A copying minor tenures an object only after it survived a minor, so a cohort full scheduled for tenured bytes is futile by the only measurement a minor has. 12_large_live_set's one cohort full was reached by 21.2 MB of copy-tenured bytes over a 16 MB bound and reclaimed 8.3 MB; every cohort full on the JSON rows was reached by in-place promotions alone. The old-reclaim baseline credit still takes every promoted byte. --- crates/perry-runtime/src/gc/copying.rs | 8 +++ crates/perry-runtime/src/gc/policy.rs | 4 +- .../perry-runtime/src/gc/promoted_cohort.rs | 50 ++++++++++++++- crates/perry-runtime/src/gc/telemetry.rs | 3 + .../src/gc/tests/promoted_cohort.rs | 61 +++++++++++++++++++ 5 files changed, 123 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index ff89e2e374..0bafd9362b 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -488,6 +488,7 @@ impl CopyingNurseryCollector { self.stats.promoted_objects += 1; self.stats.promoted_bytes += total; self.stats.in_place_promoted_objects += 1; + self.stats.in_place_promoted_bytes += total; self.live_from_bytes += total; // Survivor-influx accounting: an in-place promotion consumes the // whole young generation at once, so the split the adaptive @@ -1662,6 +1663,7 @@ pub(super) fn run_copied_minor_attempt( collector.stats.promoted_objects = promotion_stats.objects; collector.stats.in_place_promoted_objects = promotion_stats.objects; collector.stats.promoted_bytes = promotion_stats.bytes; + collector.stats.in_place_promoted_bytes = promotion_stats.bytes; collector.stats.eden_live_bytes = promotion_stats.bytes; collector.live_from_bytes = promotion_stats.bytes; } @@ -1809,6 +1811,12 @@ pub(super) fn run_copied_minor_attempt( // liveness claim, and withholding it pins that base at 0 on exactly the // workloads that reach this path. credit_promoted_bytes_to_old_baseline(collector.stats.promoted_bytes); + // #10241: the promoted cohort counts in-place promotions only; bytes this + // minor tenured by copy survived a minor already (`promoted_cohort`). + super::promoted_cohort::note_minor_promotion( + collector.stats.promoted_bytes, + collector.stats.in_place_promoted_bytes, + ); // Everything outside from-space retains its pre-minor accounting. Remove // the from-space share of that accounting, then add back exactly the // objects that survived by copy or promotion. This also preserves objects diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 347488e4f8..d5ef74ba47 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1994,8 +1994,8 @@ pub(super) fn credit_promoted_bytes_to_old_baseline(promoted_bytes: usize) { GC_LAST_OLD_RECLAIM_IN_USE_BYTES .with(|bytes| bytes.set(bytes.get().saturating_add(promoted_bytes))); // #10182: the credit hides these bytes from the growth band by design; the - // promoted-cohort bound is what still counts them. - super::promoted_cohort::note_promoted(promoted_bytes); + // promoted-cohort bound is what still counts the in-place-promoted share of + // them (`promoted_cohort::note_minor_promotion`, called by the minor). } /// Feed a copying minor's measured young-survival ratio to arena-growth pacing. diff --git a/crates/perry-runtime/src/gc/promoted_cohort.rs b/crates/perry-runtime/src/gc/promoted_cohort.rs index b8deb00671..2f5c2304de 100644 --- a/crates/perry-runtime/src/gc/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/promoted_cohort.rs @@ -42,6 +42,20 @@ //! * a cohort full that reclaims less than half the cohort doubles the bound //! (up to `BACKOFF_SHIFT_MAX`), so a retaining heap pays a logarithmic number //! of futile fulls, each O(live), and a productive full restores it. +//! +//! # Only in-place promotions count (#10241) +//! +//! A copying minor tenures an object only after it has already survived a +//! minor, so tenured bytes are live by the one measurement a minor has, and a +//! full scheduled for them is futile by the same measure. The blind spot above +//! is the in-place promotion's alone: it moves a young generation wholesale, +//! dead trees included. Measured: `12_large_live_set`'s only cohort full was +//! reached by 21.2 MB of copy-tenured bytes over a 16 MB bound and reclaimed +//! 8.3 MB (futile; wall 0.25 s → 0.28 s against base), and so was +//! `14_grow_then_churn`'s first (2.1 MB tenured, reclaimed 0), while every +//! cohort full on the JSON rows was reached by in-place promotions alone. The +//! old-reclaim baseline credit is unchanged: it still takes every promoted +//! byte. use std::cell::Cell; @@ -67,11 +81,25 @@ crate::perry_thread_local! { static COHORT_FULLS: Cell = const { Cell::new(0) }; } -/// A minor moved `bytes` into old-gen. +/// A minor moved `bytes` into old-gen by promoting its blocks in place. pub(super) fn note_promoted(bytes: usize) { PROMOTED_SINCE_FULL.with(|c| c.set(c.get().saturating_add(bytes))); } +/// A copying minor promoted `promoted` bytes, `in_place` of them by promoting +/// blocks in place and the rest by tenuring copies. The cohort takes the +/// in-place share. +pub(super) fn note_minor_promotion(promoted: usize, in_place: usize) { + #[cfg(test)] + let in_place = if sabotage::counting_tenured() { + promoted + } else { + in_place + }; + let _ = promoted; + note_promoted(in_place); +} + /// A full collection finished and verified `old_live` bytes of old-gen. pub(super) fn note_full_finished(old_live: usize) { OLD_LIVE_AT_LAST_FULL.with(|c| c.set(old_live)); @@ -144,12 +172,32 @@ pub(super) mod sabotage { thread_local! { static NEVER_BACK_OFF: Cell = const { Cell::new(false) }; + static COUNT_TENURED: Cell = const { Cell::new(false) }; } pub(super) fn never_back_off() -> bool { NEVER_BACK_OFF.with(Cell::get) } + /// The cohort takes copy-tenured bytes too, as it did before #10241. + pub(super) fn counting_tenured() -> bool { + COUNT_TENURED.with(Cell::get) + } + + pub(crate) struct CountTenuredGuard(bool); + + impl CountTenuredGuard { + pub(crate) fn arm() -> Self { + Self(COUNT_TENURED.with(|s| s.replace(true))) + } + } + + impl Drop for CountTenuredGuard { + fn drop(&mut self) { + COUNT_TENURED.with(|s| s.set(self.0)); + } + } + pub(crate) struct Guard(bool); impl Guard { diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index a6fbb58678..3f33c3614d 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -308,6 +308,9 @@ pub(super) struct CopyingNurseryTraceStats { /// row with `in_place_promotion=true` and zero here promoted nothing and /// proves nothing. pub(super) in_place_promoted_objects: usize, + /// Bytes promoted by that path — `promoted_bytes` minus what the same + /// cycle tenured by copy. The promoted cohort counts only these (#10241). + pub(super) in_place_promoted_bytes: usize, pub(super) in_place_promoted_blocks: usize, /// Bytes on the promoted blocks that were NOT live — the footprint this /// technique trades for the speed, retained until the next full. diff --git a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs index eaa451ef16..6fdb8e9b18 100644 --- a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs @@ -119,6 +119,7 @@ fn the_cohort_never_makes_old_reclaim_due() { GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|c| c.set(4 * MB)); cohort::seed_for_tests(0, 0, 0); credit_promoted_bytes_to_old_baseline(270 * MB); + cohort::note_promoted(270 * MB); assert!( cohort::full_due(), "premise: the cohort is far past its bound" @@ -193,6 +194,66 @@ fn below_the_bound_no_cohort_full_runs_and_the_dead_promoted_object_stays() { ); } +/// #10241: rooted young strings survive evacuating minors until they tenure, +/// with the cohort one byte short of its bound. Returns the bytes tenured, the +/// cohort growth, and whether the cohort full then ran. +fn tenure_into_a_cohort_one_byte_short(count_tenured: bool) -> (usize, usize, bool) { + let _guard = CopyingNurseryTestGuard::new(4); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _tenuring = crate::gc::tenuring::set_survivals_for_test(1); + let _sabotage = count_tenured.then(cohort::sabotage::CountTenuredGuard::arm); + rooted_young_strings(0, 4000); + let short = cohort::bound_bytes() - 1; + cohort::seed_for_tests(short, 0, 0); + let mut tenured = 0usize; + for _ in 0..3 { + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert!( + !trace.copying_nursery.in_place_promotion, + "premise: evacuating minors, nothing promoted in place" + ); + tenured += + trace.copying_nursery.promoted_bytes - trace.copying_nursery.in_place_promoted_bytes; + } + let grown = cohort::promoted_since_full() - short; + let ran = run_promoted_cohort_full_if_due(); + js_shadow_slot_set(0, crate::value::TAG_UNDEFINED); + cohort::seed_for_tests(0, 0, 0); + (tenured, grown, ran) +} + +#[test] +fn copy_tenured_bytes_do_not_bring_the_cohort_to_its_bound() { + let (tenured, grown, ran) = tenure_into_a_cohort_one_byte_short(false); + assert!( + tenured > 0, + "premise: the minors tenured the rooted strings" + ); + assert_eq!(grown, 0, "the cohort counts in-place promotions only"); + assert!( + !ran, + "tenured bytes survived a minor already: no full is scheduled for them \ + (12_large_live_set's futile cohort full)" + ); +} + +#[test] +fn sabotaged_tenured_accounting_schedules_a_full_for_live_survivors() { + let (tenured, grown, ran) = tenure_into_a_cohort_one_byte_short(true); + assert!( + tenured > 0, + "premise: the minors tenured the rooted strings" + ); + assert_eq!( + grown, tenured, + "sabotaged, the cohort takes the tenured bytes" + ); + assert!( + ran, + "and the byte short of the bound is made up by live survivors" + ); +} + /// What the young population promoted at the cohort's safepoint looks like at /// the full. #[derive(Clone, Copy, PartialEq)] From b13d07484c4a82a12789bf178f8bb5e1fe12bc7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 09:07:24 +0200 Subject: [PATCH 38/40] changelog: cohort survival feed and in-place-only cohort (#10182) --- changelog.d/10241-pacing-full-cost.md | 40 +++++++++++++++++++++++++++ scripts/gc_runtime_root_holders.json | 4 +-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/changelog.d/10241-pacing-full-cost.md b/changelog.d/10241-pacing-full-cost.md index 4ee44cc8cb..4b91848e7f 100644 --- a/changelog.d/10241-pacing-full-cost.md +++ b/changelog.d/10241-pacing-full-cost.md @@ -54,6 +54,46 @@ less than base's: 42 ms against 47 ms. On a real cohort full in `records_array_20m:parse` the pause is 30–32 ms, down from the 62–68 ms #10220 measured. +**Cohort survival feed** (`gc/promoted_cohort/survival.rs`): + +- **Defect.** Every full resets the untraced-promotion budget, so a cohort + full every ~live bytes kept `14_grow_then_churn` promoting its churn + untraced for good. No minor measured again: 13 cohort fulls, 0 copied + objects, wall 429 → 594 ms against #10220. +- **Measurement.** A cohort full's sweep measures how much of what the minor + at its own safepoint promoted is still live. Blocks walked whole report their + live bytes; dead blocks reclaimed unwalked count as 0. +- **When it is fed.** The figure replaces the young-survival predictor (below + 950‰) only when it equals what a minor would measure. After the mark, the + unmarked objects on the minor's dirty old pages have their dirty slots read. + A slot pointing into the promoted blocks means a dead remembered parent + held them, and then nothing is fed. +- **Why the gate.** The JSON rows measure 500‰ (8m scan: 333‰), because each + minor promotes the dead previous tree along with the live one. Their tree + arrays are exactly such dead parents. Fed unconditionally, the next minor + evacuates both trees: `records_array_20m:parse` +20.0 % CPU / +54.9 MiB, + `records_array_8m:scan` +14.1 % / +31.2 MiB (mini, best of 3). +- **Diagnostics.** `[gc-promoted-cohort]` adds `promoted_by_minor=`, + `live_of_promoted=`, `survival_permille=`, `minor_view=` and `predictor=`. + +**Only in-place promotions fill the cohort.** A copying minor tenures an object +only after it survived a minor, so a full scheduled for tenured bytes is +futile. `12_large_live_set`'s one cohort full was reached by 21.2 MB of tenured +bytes over a 16 MB bound and reclaimed 8.3 MB; every cohort full on the JSON +rows was reached by in-place promotions alone. The old-reclaim baseline credit +still takes every promoted byte. + +Laptop, interleaved, 7 rounds (median wall / peak RSS): + +| probe | #10220 | before these two changes | now | +|---|---|---|---| +| `14_grow_then_churn` | 0.44 s / 284.6 MiB | 0.63 s / 131.1 MiB | 0.29 s / 65.1 MiB | +| `12_large_live_set` | 0.26 s / 109.1 MiB | 0.28 s / 115.6 MiB | 0.24 s / 109.0 MiB | + +On the mini, every JSON row's collection schedule (minors, fulls, cohort +sizes, bounds and reclaimed bytes) is identical before and after both changes, +and CPU stays within ±1.0 %, so the table below stands. + **This does not meet #10182's acceptance bar.** Interleaved best-of-3 on the same tree, base `9a05821b9e`: diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 976884192d..d172c74801 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -313,7 +313,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes — in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes — in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -331,7 +331,7 @@ "crates/perry-runtime/src/gc/census.rs": "5c151725460ffb92a55a6bee781123ef5159263b4ce5958d16570f78216e0d67", "crates/perry-runtime/src/gc/cycle.rs": "b035dcb44df029358cbab0afaa526e8e506765f5178034663257e18ceefaf9df", "crates/perry-runtime/src/gc/mod.rs": "9fedd2790f48154aaeceefb4805d3fbaa2fdf3c407529b326425fde86c2bf9a5", - "crates/perry-runtime/src/gc/policy.rs": "853e9bf44a03d3d14a335c5c1566732d449bbb2b62da3d656f5578f8c4805ccc", + "crates/perry-runtime/src/gc/policy.rs": "aee430efe60cedec2bb5c7aba7bf3e929636145049630f2cb106cceff7613b44", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } } From f9b2f907ba915f76014526a7829a4e101e088aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 14:24:20 +0200 Subject: [PATCH 39/40] fix(gc): preserve the sabotage panic in debug and pin the merged census window --- crates/perry-runtime/src/gc/tests/sweep_described_runs.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/gc/tests/sweep_described_runs.rs b/crates/perry-runtime/src/gc/tests/sweep_described_runs.rs index 07b107bdcb..a3af926747 100644 --- a/crates/perry-runtime/src/gc/tests/sweep_described_runs.rs +++ b/crates/perry-runtime/src/gc/tests/sweep_described_runs.rs @@ -23,7 +23,7 @@ fn run_isolated(test: fn()) { test(); }) .join() - .expect("described-run sweep test thread must not panic"); + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)); } struct Planted { @@ -204,7 +204,13 @@ fn a_dead_object_on_a_described_page_leaves_the_page_accounting_exact() { }); } +// Debug builds detect the deliberately stale run before the release-only +// accounting observation below. Preserve and require that exact guard failure. #[test] +#[cfg_attr( + debug_assertions, + should_panic(expected = "a promoted page run did not re-parse to the object count") +)] fn sabotaged_expansion_order_keeps_counting_a_freed_object() { run_isolated(|| { let r = one_dead_on_the_page(true); From 73c7aaa9ed00ac56a99e3de3d4cc96828c596269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 14:24:21 +0200 Subject: [PATCH 40/40] chore: release merge train 190 as v0.5.1568 --- CLAUDE.md | 2 +- Cargo.lock | 162 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 3 files changed, 83 insertions(+), 83 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 906bd83cc2..9c84bd68e5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1567 +**Current Version:** 0.5.1568 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 2bf17339cd..534d2e3652 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5690,7 +5690,7 @@ checksum = "1542e48011813fbdf3c075da4a4ed53ee93c816eef62e36eb5064a6fd2be10a5" [[package]] name = "perry" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "base64 0.22.1", @@ -5754,7 +5754,7 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-dispatch", "serde", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-audio-miniaudio" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "cc", "libc", @@ -5771,7 +5771,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "aho-corasick", "anyhow", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "perry-hir", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "perry-hir", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "perry-dispatch", @@ -5814,7 +5814,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "perry-hir", @@ -5822,7 +5822,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "base64 0.22.1", @@ -5834,7 +5834,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "perry-hir", @@ -5842,7 +5842,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "async-trait", @@ -5870,14 +5870,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "serde", "serde_json", @@ -5885,7 +5885,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1567" +version = "0.5.1568" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "clap", @@ -5911,7 +5911,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "block2", "objc2", @@ -5921,7 +5921,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "argon2", "perry-ffi", @@ -5930,7 +5930,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "reqwest", @@ -5939,7 +5939,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "bcrypt", "perry-ffi", @@ -5947,7 +5947,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "rusqlite", @@ -5955,7 +5955,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "scraper", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "perry-runtime", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "chrono", "cron", @@ -5981,7 +5981,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "chrono", "perry-ffi", @@ -5989,7 +5989,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "rust_decimal", @@ -5997,7 +5997,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "serde_json", @@ -6005,7 +6005,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "rand 0.10.2", @@ -6013,7 +6013,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "perry-runtime", @@ -6021,14 +6021,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "bytes", "http-body-util", @@ -6046,7 +6046,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "bytes", "lazy_static", @@ -6059,7 +6059,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "bytes", @@ -6091,7 +6091,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "lazy_static", "perry-ffi", @@ -6101,7 +6101,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "jsonwebtoken", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "lru", "perry-ffi", @@ -6121,7 +6121,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "chrono", "perry-ffi", @@ -6129,7 +6129,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "bson", "futures-util", @@ -6141,7 +6141,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "chrono", "perry-ffi", @@ -6153,7 +6153,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "nanoid", "perry-ffi", @@ -6162,7 +6162,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "bytes", "perry-ffi", @@ -6177,7 +6177,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "const-oid 0.10.2", "der 0.8.1", @@ -6196,7 +6196,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "lettre", "perry-ffi", @@ -6206,7 +6206,7 @@ dependencies = [ [[package]] name = "perry-ext-parcel-watcher" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "notify", "perry-ffi", @@ -6218,7 +6218,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "printpdf", @@ -6226,7 +6226,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "sqlx", @@ -6235,7 +6235,7 @@ dependencies = [ [[package]] name = "perry-ext-qs" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "perry-runtime", @@ -6244,7 +6244,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "governor", "perry-ffi", @@ -6252,7 +6252,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "fast_image_resize", "image", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "perry-ext-streams" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "lazy_static", "perry-ffi", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "perry-ext-typescript" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "perry-ffi", @@ -6292,7 +6292,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "perry-runtime", @@ -6301,7 +6301,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "uuid", @@ -6309,7 +6309,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-ffi", "perry-validation", @@ -6318,7 +6318,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "futures-util", "lazy_static", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "brotli", "flate2", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "dashmap 6.2.1", "once_cell", @@ -6351,7 +6351,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "perry-api-manifest", @@ -6372,11 +6372,11 @@ dependencies = [ [[package]] name = "perry-native-registration" -version = "0.5.1567" +version = "0.5.1568" [[package]] name = "perry-parser" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "perry-diagnostics", @@ -6390,7 +6390,7 @@ dependencies = [ [[package]] name = "perry-perex" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perex", "regex", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "ahash", "anyhow", @@ -6458,14 +6458,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6560,14 +6560,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "perry-hir", @@ -6576,7 +6576,7 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "dirs", "perry-ffi", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "perry-ui-android" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "itoa", @@ -6604,7 +6604,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "rand 0.10.2", "serde", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "cairo-rs 0.22.9", @@ -6637,7 +6637,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "block2", @@ -6654,7 +6654,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "block2", @@ -6671,7 +6671,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1567" +version = "0.5.1568" [[package]] name = "perry-ui-test" @@ -6682,11 +6682,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1567" +version = "0.5.1568" [[package]] name = "perry-ui-tvos" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "block2", @@ -6703,7 +6703,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "block2", @@ -6720,7 +6720,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "block2", "libc", @@ -6734,7 +6734,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "libc", @@ -6753,7 +6753,7 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "base64 0.22.1", "libc", @@ -6766,7 +6766,7 @@ dependencies = [ [[package]] name = "perry-updater" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "anyhow", "base64 0.22.1", @@ -6782,7 +6782,7 @@ dependencies = [ [[package]] name = "perry-validation" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "idna", "regex", @@ -6792,7 +6792,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1567" +version = "0.5.1568" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index ddf7dc64b5..d226df4962 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,7 +338,7 @@ codegen-units = 1 codegen-units = 1 [workspace.package] -version = "0.5.1567" +version = "0.5.1568" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry"