diff --git a/CLAUDE.md b/CLAUDE.md index 6d9da404ed..5db1926fa3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1561 +**Current Version:** 0.5.1562 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 5308f7bc18..3d2aa7f6a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5690,7 +5690,7 @@ checksum = "1542e48011813fbdf3c075da4a4ed53ee93c816eef62e36eb5064a6fd2be10a5" [[package]] name = "perry" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "base64 0.22.1", @@ -5754,7 +5754,7 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-dispatch", "serde", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-audio-miniaudio" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "cc", "libc", @@ -5771,7 +5771,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "aho-corasick", "anyhow", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "perry-hir", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "perry-hir", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "perry-dispatch", @@ -5814,7 +5814,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "perry-hir", @@ -5822,7 +5822,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "base64 0.22.1", @@ -5834,7 +5834,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "perry-hir", @@ -5842,7 +5842,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "async-trait", @@ -5870,14 +5870,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "serde", "serde_json", @@ -5885,7 +5885,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1561" +version = "0.5.1562" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "clap", @@ -5911,7 +5911,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "block2", "objc2", @@ -5921,7 +5921,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "argon2", "perry-ffi", @@ -5930,7 +5930,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "reqwest", @@ -5939,7 +5939,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "bcrypt", "perry-ffi", @@ -5947,7 +5947,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "rusqlite", @@ -5955,7 +5955,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "scraper", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "perry-runtime", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "chrono", "cron", @@ -5981,7 +5981,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "chrono", "perry-ffi", @@ -5989,7 +5989,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "rust_decimal", @@ -5997,7 +5997,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "serde_json", @@ -6005,7 +6005,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "rand 0.10.2", @@ -6013,7 +6013,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "perry-runtime", @@ -6021,14 +6021,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "bytes", "http-body-util", @@ -6046,7 +6046,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "bytes", "lazy_static", @@ -6059,7 +6059,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "bytes", @@ -6091,7 +6091,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "lazy_static", "perry-ffi", @@ -6101,7 +6101,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "jsonwebtoken", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "lru", "perry-ffi", @@ -6121,7 +6121,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "chrono", "perry-ffi", @@ -6129,7 +6129,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "bson", "futures-util", @@ -6141,7 +6141,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "chrono", "perry-ffi", @@ -6153,7 +6153,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "nanoid", "perry-ffi", @@ -6162,7 +6162,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "bytes", "perry-ffi", @@ -6177,7 +6177,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "const-oid 0.10.2", "der 0.8.1", @@ -6196,7 +6196,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "lettre", "perry-ffi", @@ -6206,7 +6206,7 @@ dependencies = [ [[package]] name = "perry-ext-parcel-watcher" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "notify", "perry-ffi", @@ -6218,7 +6218,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "printpdf", @@ -6226,7 +6226,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "sqlx", @@ -6235,7 +6235,7 @@ dependencies = [ [[package]] name = "perry-ext-qs" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "perry-runtime", @@ -6244,7 +6244,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "governor", "perry-ffi", @@ -6252,7 +6252,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "fast_image_resize", "image", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "perry-ext-streams" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "lazy_static", "perry-ffi", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "perry-ext-typescript" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "perry-ffi", @@ -6292,7 +6292,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "perry-runtime", @@ -6301,7 +6301,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "uuid", @@ -6309,7 +6309,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "perry-validation", @@ -6318,7 +6318,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "futures-util", "lazy_static", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "brotli", "flate2", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "dashmap 6.2.1", "once_cell", @@ -6351,7 +6351,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "perry-api-manifest", @@ -6372,11 +6372,11 @@ dependencies = [ [[package]] name = "perry-native-registration" -version = "0.5.1561" +version = "0.5.1562" [[package]] name = "perry-parser" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "perry-diagnostics", @@ -6390,7 +6390,7 @@ dependencies = [ [[package]] name = "perry-perex" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perex", "regex", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "ahash", "anyhow", @@ -6458,14 +6458,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6560,14 +6560,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "perry-hir", @@ -6576,7 +6576,7 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "perry-ffi", "perry-ui-model", @@ -6584,7 +6584,7 @@ dependencies = [ [[package]] name = "perry-ui-android" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "itoa", @@ -6602,7 +6602,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "rand 0.10.2", "serde", @@ -6612,7 +6612,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "cairo-rs 0.22.9", @@ -6635,7 +6635,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "block2", @@ -6652,7 +6652,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "block2", @@ -6669,7 +6669,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1561" +version = "0.5.1562" [[package]] name = "perry-ui-test" @@ -6680,11 +6680,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1561" +version = "0.5.1562" [[package]] name = "perry-ui-tvos" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "block2", @@ -6701,7 +6701,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "block2", @@ -6718,7 +6718,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "block2", "libc", @@ -6732,7 +6732,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "libc", @@ -6751,7 +6751,7 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "base64 0.22.1", "libc", @@ -6764,7 +6764,7 @@ dependencies = [ [[package]] name = "perry-updater" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "anyhow", "base64 0.22.1", @@ -6780,7 +6780,7 @@ dependencies = [ [[package]] name = "perry-validation" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "idna", "regex", @@ -6790,7 +6790,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1561" +version = "0.5.1562" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index 4c0035c9b7..10b26537d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,7 +338,7 @@ codegen-units = 1 codegen-units = 1 [workspace.package] -version = "0.5.1561" +version = "0.5.1562" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry" diff --git a/changelog.d/10217-gc-block-granular-sweep.md b/changelog.d/10217-gc-block-granular-sweep.md new file mode 100644 index 0000000000..32b9d38f0c --- /dev/null +++ b/changelog.d/10217-gc-block-granular-sweep.md @@ -0,0 +1,28 @@ +A synchronous full mark-sweep now reclaims an arena block without walking its +objects when the cycle's exact pointer census shows the trace reached nothing +in it and nothing in it owes per-object sweep work (no finalizer, no pinned, +forwarded or pre-marked header, no raw-f64 array layout bits, and no +address-keyed side-table entry the dead-owner prune does not already drop). +The census seals its address-ordered runs at block boundaries, so a successful +membership query names the reached block at no extra lookup; the block cleanup +still resets or releases the block and drops its old-page index, and freed +bytes come from the census's per-block sums. The require-marked old-to-young +remembered-set rebuild skips the same unreached blocks. Budgeted fulls and +minors keep the per-object walk. `PERRY_GC_DIAG=1` prints +`block_skip_reclaimed_blocks/_objects/_bytes` on each `[gc] blocks:` line. + +Measured on a loop that parses `records_array_8m.json`, keeps one tree, and +calls `gc()` each iteration: sweep 5.4 → 2.6 ms and 12–15 → 3.7–4.0 ms on the +fulls that follow a minor, atomic finalize 3.3 → 2.2 ms, census 2.7 → 3.3 ms; +the six fulls total 159 ms against 203 ms on the base. gc-ratchet gated +counters are unchanged on all 14 probes (block skip is live on 08, 09, 11 and +14), and the 22-row JSON matrix is unchanged within noise because none of its +rows runs a full collection on `main`. + +The #10182 pacing half — an old-reclaim trigger bounded by the promoted cohort +— was measured and not included: every variant that produced the intended +regime (a full every two to three parses, peak RSS at or below Node's) cost ++136 to +304 ms of CPU on the target rows, far beyond their CPU lead. A full +over one live 20 MB tree costs ~100 ms even with dead blocks skipped: mark +45 ms, remembered-set rebuild 21 ms, per-live-object sweep accounting 16–19 ms, +census 10–15 ms. diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index ad3b0bf4e1..1fc2224482 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -112,7 +112,7 @@ pub(crate) use reset::{ copying_active_survivor_in_use_bytes, copying_from_space_in_use_bytes, copying_prepare_to_space, copying_reset_from_spaces_and_flip, old_arena_reclaim_dead_blocks, old_arena_reclaim_selected_dead_blocks, survivor_arena_reclaim_dead_blocks, - ArenaResetEmptyBlocksState, OldArenaReclaimDeadBlocksState, + survivor_block_index_range, ArenaResetEmptyBlocksState, OldArenaReclaimDeadBlocksState, SurvivorArenaReclaimDeadBlocksState, }; pub use reset::{arena_reset_all_blocks_to_zero, arena_reset_empty_blocks}; diff --git a/crates/perry-runtime/src/arena/reset.rs b/crates/perry-runtime/src/arena/reset.rs index 935be06cce..faf9525426 100644 --- a/crates/perry-runtime/src/arena/reset.rs +++ b/crates/perry-runtime/src/arena/reset.rs @@ -154,6 +154,15 @@ pub(crate) fn block_in_copying_from_space( block_idx < general_n || active_survivor.contains(&block_idx) } +/// Global block indices of both survivor arenas (the region between the +/// general arena and the longlived arena). +pub(crate) fn survivor_block_index_range() -> std::ops::Range { + let general_n = ARENA.with(|a| unsafe { (*a.get()).blocks.len() }); + let survivor0_n = SURVIVOR_ARENA_0.with(|a| unsafe { (*a.get()).blocks.len() }); + let survivor1_n = SURVIVOR_ARENA_1.with(|a| unsafe { (*a.get()).blocks.len() }); + general_n..general_n + survivor0_n + survivor1_n +} + pub(crate) fn active_survivor_block_index_range() -> std::ops::Range { let general_n = ARENA.with(|a| unsafe { (*a.get()).blocks.len() }); let survivor0_n = SURVIVOR_ARENA_0.with(|a| unsafe { (*a.get()).blocks.len() }); diff --git a/crates/perry-runtime/src/arena/walk.rs b/crates/perry-runtime/src/arena/walk.rs index a460808116..7f44b82f06 100644 --- a/crates/perry-runtime/src/arena/walk.rs +++ b/crates/perry-runtime/src/arena/walk.rs @@ -33,6 +33,9 @@ pub(crate) struct ArenaObjectCursor { block_pos: usize, offset: usize, finished: bool, + /// Global block indices this cursor never enters (#10182 block-granular + /// sweep). Empty for every other walker. + skip_blocks: Vec, } enum ArenaObjectCursorBlocks { @@ -110,6 +113,7 @@ impl ArenaObjectCursorBuilder { block_pos: 0, offset: 0, finished: false, + skip_blocks: Vec::new(), }); } @@ -264,6 +268,20 @@ impl ArenaObjectCursor { self.finished } + /// Never enter the blocks whose global index is set in `skip` (#10182). + /// Must be installed before the first `next`; a block the cursor is + /// already inside is not affected. + pub(crate) fn set_skip_blocks(&mut self, skip: Vec) { + self.skip_blocks = skip; + } + + /// `(global block index, data, offset)` of the block the last yielded + /// object came from, as snapshotted when the cursor was built. + pub(crate) fn current_block_extent(&self) -> Option<(usize, usize, usize)> { + self.current_block + .map(|block| (block.block_idx, block.data, block.offset)) + } + pub(crate) fn next(&mut self) -> Option<(*mut u8, usize)> { let mut remaining = usize::MAX; self.next_budgeted(&mut remaining) @@ -273,21 +291,31 @@ impl ArenaObjectCursor { if self.current_block.is_some() { return true; } - self.current_block = match &mut self.blocks { - ArenaObjectCursorBlocks::BlockIndex(blocks) => { - let block = blocks.get(self.block_pos).copied(); - if block.is_some() { - self.block_pos += 1; + loop { + self.current_block = match &mut self.blocks { + ArenaObjectCursorBlocks::BlockIndex(blocks) => { + let block = blocks.get(self.block_pos).copied(); + if block.is_some() { + self.block_pos += 1; + } + block } - block + ArenaObjectCursorBlocks::Address(blocks) => blocks.next(), + }; + let Some(block) = self.current_block else { + self.finished = true; + return false; + }; + if !self + .skip_blocks + .get(block.block_idx) + .copied() + .unwrap_or(false) + { + return true; } - ArenaObjectCursorBlocks::Address(blocks) => blocks.next(), - }; - if self.current_block.is_none() { - self.finished = true; - return false; + self.current_block = None; } - true } } diff --git a/crates/perry-runtime/src/array/element_shape.rs b/crates/perry-runtime/src/array/element_shape.rs index 1fa9316ca2..d55ace9f6a 100644 --- a/crates/perry-runtime/src/array/element_shape.rs +++ b/crates/perry-runtime/src/array/element_shape.rs @@ -852,6 +852,13 @@ pub(crate) fn test_element_shape_record_exists(owner: usize) -> bool { ELEMENT_SHAPES.with(|m| m.borrow().contains_key(&owner)) } +/// Plant a record (and its advertising bit) for `owner` without verifying any +/// elements — a fixture for the collector's side-table tests. +#[cfg(test)] +pub(crate) fn test_seed_element_shape_record(owner: usize) { + unsafe { establish(owner as *mut ArrayHeader, 1, 0, 0) }; +} + #[cfg(test)] pub(crate) fn test_clear_element_shape_table() { ELEMENT_SHAPES.with(|m| m.borrow_mut().clear()); diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index fc2c981e47..5231329d55 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -100,7 +100,9 @@ pub use self::element_shape::{ js_array_element_shape_version, js_array_ensure_element_shape, }; #[cfg(test)] -pub(crate) use self::element_shape::{test_element_shape_record_exists, test_serialize}; +pub(crate) use self::element_shape::{ + test_element_shape_record_exists, test_seed_element_shape_record, test_serialize, +}; pub use self::flat_clone::{ js_array_clone, js_array_clone_for_spread, js_array_entries, js_array_flat, js_array_flat_depth, js_array_keys, js_array_values, js_arraylike_flat, diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index db975cf290..826ea115d9 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1283,12 +1283,25 @@ impl GcCycleState { return; } let done = { + // #10182: a synchronous full's census knows which blocks + // the trace never reached; the require-marked walk skips + // them. A budgeted cycle has no census (and its mutator + // windows can still shade), so it walks everything. + let budgeted = self.progress_kind.is_budgeted(); + let valid_ptrs = self.valid_ptrs.as_ref(); let state = self .atomic_finalize .as_mut() .expect("atomic finalize state exists"); let rebuild = state.remembered_rebuild.get_or_insert_with(|| { - OldToYoungRememberedRebuildState::new(/* require_marked = */ true) + let skip = if budgeted { + None + } else { + valid_ptrs.and_then(|ptrs| ptrs.block_census.unmarked_blocks()) + }; + OldToYoungRememberedRebuildState::new_skipping( + /* require_marked = */ true, skip, + ) }); rebuild.step(budget) }; @@ -1517,6 +1530,16 @@ impl GcCycleState { full_trace && !self.progress_kind.is_budgeted(), ), ); + // #10182: a synchronous full reclaims dead, obligation-free blocks + // without walking them. Only its census-built pointer set records + // which blocks the trace reached; a budgeted cycle's classifier + // set is disarmed and this is a no-op. + if full_trace && !self.progress_kind.is_budgeted() { + if let Some(valid_ptrs) = self.valid_ptrs.as_ref() { + let sweep = self.sweep_state.take().expect("sweep state was just built"); + self.sweep_state = Some(sweep.with_block_skip(&valid_ptrs.block_census)); + } + } } let done = self .sweep_state diff --git a/crates/perry-runtime/src/gc/oldgen.rs b/crates/perry-runtime/src/gc/oldgen.rs index b0852e72ce..3a331d76a8 100644 --- a/crates/perry-runtime/src/gc/oldgen.rs +++ b/crates/perry-runtime/src/gc/oldgen.rs @@ -1241,6 +1241,14 @@ impl IncrementalSweepState { self } + /// #10182: let the arena walk reclaim whole dead blocks without visiting + /// them, from the synchronous full cycle's census. See + /// `ArenaSweepObjectsState::apply_block_skip`. + pub(super) fn with_block_skip(mut self, census: &super::trace::BlockCensus) -> Self { + self.arena.apply_block_skip(census); + self + } + pub(super) fn step(&mut self, budget: usize) -> bool { match self.subphase { SweepCycleSubphase::CollectionSideBuffers => { @@ -1345,318 +1353,6 @@ impl IncrementalSweepState { } } -struct ArenaSweepObjectsState { - cursor: crate::arena::ArenaObjectCursor, - /// Dead old headers awaiting one batched page-index removal (see `sweep_batch`). - pending_old_unregister: sweep_batch::PendingOldUnregister, - block_snapshots: Vec, - block_has_live: Vec, - resettable_general_n: usize, - old_block_start: usize, - overflow_active: bool, - do_age_bump: bool, - reclaim_dead_old_blocks: bool, - /// This sweep follows a MINOR trace, whose mark bits say nothing about the - /// old generation: old-gen parents are black leaves whose slots are only - /// visited through dirty remembered-set pages, so an object reachable only - /// from a non-dirty old parent is never marked. "Unmarked" therefore does - /// NOT imply "dead" for anything in the old generation. - /// - /// Two consequences, both handled below: - /// - /// * Forwarding stubs must ALL be retained: array growth installs - /// PERMANENT stubs (#6228 — stale pre-growth pointers keep resolving for - /// reads, references are never rewritten). An old parent (e.g. a - /// long-lived Map's entries buffer) whose page is no longer dirty never - /// marks the stub its slot points at, so reclaiming it is a - /// use-after-free. - /// * Ordinary old-gen objects must not be reclaimed either (#6892). The - /// minor never frees their memory, but `reclaim_dead_object` still runs - /// `finalize_dead_arena_payload` on them, which wipes a LIVE object's GC - /// slot-layout mask and payload side tables and frees its external - /// payload buffers. - /// - /// Full traces DO visit every live parent, so mark-based reclaim stays - /// sound there (and bounds the accumulation). - minor_sweep: bool, - /// Old-gen blocks selected for page defrag this cycle. Every indexed - /// occupant was evacuated out during this same cycle, so what is left - /// really is reclaimable even in a minor — and the block-level reclaim - /// needs `block_has_live` to stay false for them. - targeted_old_blocks: Option>, - freed_bytes: u64, - retained_forwarded_stub_objects: usize, - retained_forwarded_stub_bytes: usize, - /// #7598 Eden census: see `SweepTraceStats`. - eden_live_bytes: u64, - eden_dead_bytes: u64, - arena_live_bytes: u64, - /// #7901: see `SweepTraceStats::arena_live_from_space_bytes`. - arena_live_from_space_bytes: u64, - active_survivor_blocks: std::ops::Range, -} - -impl ArenaSweepObjectsState { - fn new( - do_age_bump: bool, - reclaim_dead_old_blocks: bool, - minor_sweep: bool, - targeted_old_blocks: Option>, - ) -> Self { - let n_blocks = crate::arena::arena_block_count(); - let block_snapshots = crate::arena::arena_block_snapshots(); - crate::arena::old_pages_reset_sweep_accounting(); - Self { - cursor: crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::BlockIndex), - pending_old_unregister: Default::default(), - block_snapshots, - block_has_live: vec![false; n_blocks], - resettable_general_n: crate::arena::general_block_count(), - old_block_start: crate::arena::longlived_end(), - // Wave 2: also arms the closure dynamic-props dead-payload arm - // (one gate check per sweep-state build, not per object). - overflow_active: !crate::object::overflow_fields_is_empty() - || crate::closure::closure_dynamic_side_tables_nonempty(), - do_age_bump, - reclaim_dead_old_blocks, - minor_sweep, - targeted_old_blocks, - freed_bytes: 0, - retained_forwarded_stub_objects: 0, - retained_forwarded_stub_bytes: 0, - eden_live_bytes: 0, - eden_dead_bytes: 0, - arena_live_bytes: 0, - arena_live_from_space_bytes: 0, - active_survivor_blocks: crate::arena::active_survivor_block_index_range(), - } - } - - /// #7437: rebuild the old-gen hole free list once the object walk - /// completes — block liveness is final at that point, and the block - /// cleanup that follows only touches blocks with NO live object, which - /// the rebuild's filter already skips. - fn push_live_block_holes(&mut self) { - if self.reclaim_dead_old_blocks { - super::old_free_rebuild_from_live_old_blocks( - &self.block_has_live, - self.old_block_start, - ); - if crate::gc::gc_diag_enabled() { - eprintln!("[gc-old-free] reusable_bytes={}", super::old_free_bytes()); - } - } - } - - fn step(&mut self, budget: usize) -> bool { - let mut remaining = budget; - let mut done = false; - while remaining > 0 { - let Some((header_ptr, block_idx)) = self.cursor.next() else { - done = true; - break; - }; - remaining -= 1; - self.process_object(header_ptr as *mut GcHeader, block_idx); - } - // Never leave a dead header in the page index across a step boundary. - self.pending_old_unregister.flush(); - done - } - - fn block_has_live(&self) -> &[bool] { - &self.block_has_live - } - - fn block_snapshots(&self) -> &[crate::arena::ArenaBlockSnapshot] { - &self.block_snapshots - } - - fn maybe_print_diag(&self) { - if !crate::gc::gc_diag_enabled() { - return; - } - let live_general = (0..self.resettable_general_n) - .filter(|&i| self.block_has_live[i]) - .count(); - let live_ll = (self.resettable_general_n..self.block_has_live.len()) - .filter(|&i| self.block_has_live[i]) - .count(); - eprintln!( - "[gc] blocks: general={} ({} live), non_general={} ({} live, survivors+longlived+old), freed_bytes={} retained_forwarded_stub_bytes={} retained_forwarded_stub_objects={}", - self.resettable_general_n, - live_general, - self.block_has_live.len() - self.resettable_general_n, - live_ll, - self.freed_bytes, - self.retained_forwarded_stub_bytes, - self.retained_forwarded_stub_objects, - ); - } - - fn process_object(&mut self, header: *mut GcHeader, block_idx: usize) { - unsafe { - let age_bump_this = self.do_age_bump && block_idx < self.resettable_general_n; - let flags = (*header).gc_flags; - if flags == 0 { - self.reclaim_dead_object(header, block_idx); - return; - } - if flags & GC_FLAG_PINNED != 0 { - self.keep_live_object(header, block_idx, flags, age_bump_this, true, true); - return; - } - if flags & GC_FLAG_FORWARDED != 0 { - self.process_forwarded_object(header, block_idx, flags); - return; - } - if flags & GC_FLAG_MARKED == 0 && self.unmarked_is_provably_dead(block_idx) { - self.reclaim_dead_object(header, block_idx); - } else { - self.keep_live_object(header, block_idx, flags, age_bump_this, false, true); - } - } - } - - /// Does `flags & MARKED == 0` actually prove this object is garbage? - /// - /// Only when the trace that produced the marks covered the object's - /// generation. A minor trace never marks the old generation (see - /// `minor_sweep`), so an unmarked old-gen object is merely *unvisited* — - /// it stays live and must not be finalized. #6892: reclaiming one wiped - /// the GC slot-layout mask of a live old-gen array, after which the next - /// `layout_note_slot` rebuilt the mask from a single slot and the - /// following minor stopped tracing the array's other pointer elements, - /// sweeping objects that were still referenced. - /// - /// The old-page defrag targets are exempt: this cycle evacuated every - /// indexed occupant, so the remainder is genuinely reclaimable. - #[inline] - fn unmarked_is_provably_dead(&self, block_idx: usize) -> bool { - if !self.minor_sweep || block_idx < self.old_block_start { - return true; - } - self.targeted_old_blocks - .as_ref() - .is_some_and(|selected| selected.contains(&block_idx)) - } -} - -impl ArenaSweepObjectsState { - unsafe fn keep_live_object( - &mut self, - header: *mut GcHeader, - block_idx: usize, - flags: u8, - age_bump_this: bool, - pinned: bool, - count_in_live_census: bool, - ) { - if block_idx >= self.old_block_start { - crate::arena::old_page_account_swept_object( - header as usize, - (*header).size as usize, - true, - pinned, - ); - } - if block_idx < self.block_has_live.len() { - self.block_has_live[block_idx] = true; - } - if block_idx < self.resettable_general_n { - self.eden_live_bytes = self.eden_live_bytes.saturating_add((*header).size as u64); - } - if count_in_live_census { - let size = (*header).size as u64; - self.arena_live_bytes = self.arena_live_bytes.saturating_add(size); - // #7901: the from-space share of the census, so a following copied - // minor can remove exactly what it replaces. - if crate::arena::block_in_copying_from_space( - block_idx, - self.resettable_general_n, - &self.active_survivor_blocks, - ) { - self.arena_live_from_space_bytes = - self.arena_live_from_space_bytes.saturating_add(size); - } - } - if age_bump_this && flags & GC_FLAG_TENURED == 0 { - if flags & GC_FLAG_HAS_SURVIVED != 0 { - (*header).gc_flags = - (flags | GC_FLAG_TENURED) & !GC_FLAG_HAS_SURVIVED & !GC_FLAG_MARKED; - } else { - (*header).gc_flags = (flags | GC_FLAG_HAS_SURVIVED) & !GC_FLAG_MARKED; - } - } else { - (*header).gc_flags = flags & !GC_FLAG_MARKED; - } - } - - unsafe fn process_forwarded_object( - &mut self, - header: *mut GcHeader, - block_idx: usize, - flags: u8, - ) { - // See `minor_sweep`: a minor cannot prove a stub unreferenced (old-gen - // parents are black leaves), so it must keep them all; a full trace - // reclaims the genuinely unreferenced ones. - let retain_stub = self.minor_sweep - || flags & GC_FLAG_MARKED != 0 - || (block_idx < self.resettable_general_n - && crate::arena::general_block_in_recent_window(block_idx)); - if retain_stub { - // A full collection can leave an unmarked stub in the recent-block - // safety window. It still pins the block, but it is proven dead and - // therefore excluded from live-allocation accounting. - let count_in_live_census = self.minor_sweep || flags & GC_FLAG_MARKED != 0; - self.keep_live_object(header, block_idx, flags, false, false, count_in_live_census); - if block_idx < self.resettable_general_n { - self.retained_forwarded_stub_objects = - self.retained_forwarded_stub_objects.saturating_add(1); - self.retained_forwarded_stub_bytes = self - .retained_forwarded_stub_bytes - .saturating_add((*header).size as usize); - } - return; - } - - let total_size = (*header).size as usize; - let dead_old = block_idx >= self.old_block_start; - if dead_old { - crate::arena::old_page_account_swept_object(header as usize, total_size, false, false); - } - let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE); - self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64); - layout_clear_for_ptr(user_ptr as usize); - if self.overflow_active { - gc_type_clear_dead_payload_side_tables((*header).obj_type, user_ptr as usize); - } - if self.reclaim_dead_old_blocks && dead_old { - self.pending_old_unregister.defer(header, total_size); - } else { - (*header).gc_flags = flags & !(GC_FLAG_FORWARDED | GC_FLAG_MARKED); - } - } - - unsafe fn reclaim_dead_object(&mut self, header: *mut GcHeader, block_idx: usize) { - let total_size = (*header).size as usize; - let dead_old = block_idx >= self.old_block_start; - if dead_old { - crate::arena::old_page_account_swept_object(header as usize, total_size, false, false); - } - let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE); - self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64); - if block_idx < self.resettable_general_n { - self.eden_dead_bytes = self.eden_dead_bytes.saturating_add(total_size as u64); - } - finalize_dead_arena_payload(header, user_ptr, self.overflow_active); - if self.reclaim_dead_old_blocks && dead_old { - self.pending_old_unregister.defer(header, total_size); - } - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum ArenaSweepCleanupSubphase { General, @@ -1667,8 +1363,10 @@ enum ArenaSweepCleanupSubphase { mod sweep_batch; mod sweep_cleanup; +mod sweep_objects; use super::heap_generation::{HeapChange, HeapChangeKind}; use sweep_cleanup::*; +use sweep_objects::ArenaSweepObjectsState; fn add_reset_stats( lhs: crate::arena::ArenaResetStats, diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs new file mode 100644 index 0000000000..b488837772 --- /dev/null +++ b/crates/perry-runtime/src/gc/oldgen/sweep_objects.rs @@ -0,0 +1,438 @@ +//! The arena object walk of a sweep, split from `oldgen.rs` for the 2000-line +//! file cap when #10182 added block-granular reclamation to it. + +use super::*; + +pub(super) struct ArenaSweepObjectsState { + cursor: crate::arena::ArenaObjectCursor, + /// Dead old headers awaiting one batched page-index removal (see `sweep_batch`). + pending_old_unregister: super::sweep_batch::PendingOldUnregister, + block_snapshots: Vec, + block_has_live: Vec, + resettable_general_n: usize, + old_block_start: usize, + overflow_active: bool, + do_age_bump: bool, + reclaim_dead_old_blocks: bool, + /// This sweep follows a MINOR trace, whose mark bits say nothing about the + /// old generation: old-gen parents are black leaves whose slots are only + /// visited through dirty remembered-set pages, so an object reachable only + /// from a non-dirty old parent is never marked. "Unmarked" therefore does + /// NOT imply "dead" for anything in the old generation. + /// + /// Two consequences, both handled below: + /// + /// * Forwarding stubs must ALL be retained: array growth installs + /// PERMANENT stubs (#6228 — stale pre-growth pointers keep resolving for + /// reads, references are never rewritten). An old parent (e.g. a + /// long-lived Map's entries buffer) whose page is no longer dirty never + /// marks the stub its slot points at, so reclaiming it is a + /// use-after-free. + /// * Ordinary old-gen objects must not be reclaimed either (#6892). The + /// minor never frees their memory, but `reclaim_dead_object` still runs + /// `finalize_dead_arena_payload` on them, which wipes a LIVE object's GC + /// slot-layout mask and payload side tables and frees its external + /// payload buffers. + /// + /// Full traces DO visit every live parent, so mark-based reclaim stays + /// sound there (and bounds the accumulation). + minor_sweep: bool, + /// Old-gen blocks selected for page defrag this cycle. Every indexed + /// occupant was evacuated out during this same cycle, so what is left + /// really is reclaimable even in a minor — and the block-level reclaim + /// needs `block_has_live` to stay false for them. + targeted_old_blocks: Option>, + pub(super) freed_bytes: u64, + pub(super) retained_forwarded_stub_objects: usize, + pub(super) retained_forwarded_stub_bytes: usize, + /// #7598 Eden census: see `SweepTraceStats`. + pub(super) eden_live_bytes: u64, + pub(super) eden_dead_bytes: u64, + pub(super) arena_live_bytes: u64, + /// #7901: see `SweepTraceStats::arena_live_from_space_bytes`. + pub(super) arena_live_from_space_bytes: u64, + active_survivor_blocks: std::ops::Range, + /// #10182: blocks this sweep reclaims without entering, and what they held. + block_skip_blocks: u64, + block_skip_objects: u64, + block_skip_bytes: u64, +} + +impl ArenaSweepObjectsState { + pub(super) fn new( + do_age_bump: bool, + reclaim_dead_old_blocks: bool, + minor_sweep: bool, + targeted_old_blocks: Option>, + ) -> Self { + let n_blocks = crate::arena::arena_block_count(); + let block_snapshots = crate::arena::arena_block_snapshots(); + crate::arena::old_pages_reset_sweep_accounting(); + Self { + cursor: crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::BlockIndex), + pending_old_unregister: Default::default(), + block_snapshots, + block_has_live: vec![false; n_blocks], + resettable_general_n: crate::arena::general_block_count(), + old_block_start: crate::arena::longlived_end(), + // Wave 2: also arms the closure dynamic-props dead-payload arm + // (one gate check per sweep-state build, not per object). + overflow_active: !crate::object::overflow_fields_is_empty() + || crate::closure::closure_dynamic_side_tables_nonempty(), + do_age_bump, + reclaim_dead_old_blocks, + minor_sweep, + targeted_old_blocks, + freed_bytes: 0, + retained_forwarded_stub_objects: 0, + retained_forwarded_stub_bytes: 0, + eden_live_bytes: 0, + eden_dead_bytes: 0, + arena_live_bytes: 0, + arena_live_from_space_bytes: 0, + active_survivor_blocks: crate::arena::active_survivor_block_index_range(), + block_skip_blocks: 0, + block_skip_objects: 0, + block_skip_bytes: 0, + } + } + + /// #10182: reclaim every dead, obligation-free block without visiting its + /// objects. See `gc::trace::block_skip` for the soundness argument; this + /// is the half that decides, per block of THIS sweep's snapshot: + /// + /// * the sweep follows a full trace that reclaims dead old blocks (a + /// minor's marks say nothing about old-gen, and a targeted defrag sweep + /// keeps its own per-object accounting); + /// * the census walked the same block (`data`) up to the same bump offset, + /// so nothing was born into it after the census; + /// * no census hit landed in it and no header in it owed per-object work; + /// * the cleanup that follows resets or releases the block when it has no + /// live object: general blocks outside the recent window, survivor + /// blocks, and old blocks. The recent general window keeps its dead + /// blocks across a sweep and the longlived arena has no dead-block + /// cleanup, so both keep the per-object path. + /// + /// A skipped block contributes nothing to `block_has_live`, which is the + /// only liveness the cleanup reads. + pub(super) fn apply_block_skip(&mut self, census: &super::super::trace::BlockCensus) { + if self.minor_sweep + || !self.reclaim_dead_old_blocks + || self.targeted_old_blocks.is_some() + || !census.is_armed() + { + return; + } + let survivors = crate::arena::survivor_block_index_range(); + let mut skip = vec![false; self.block_snapshots.len()]; + let mut any = false; + for (block_idx, snapshot) in self.block_snapshots.iter().enumerate() { + if snapshot.data == 0 || snapshot.offset == 0 { + continue; + } + let cleanup_reclaims_dead_block = if block_idx < self.resettable_general_n { + !crate::arena::general_block_in_recent_window(block_idx) + } else { + survivors.contains(&block_idx) || block_idx >= self.old_block_start + }; + if !cleanup_reclaims_dead_block { + continue; + } + let Some(block) = census.block(block_idx) else { + continue; + }; + #[cfg(not(test))] + let (obligation, reached) = (block.obligation, census.reached(block_idx)); + #[cfg(test)] + let (obligation, reached) = { + use super::super::trace::block_skip::sabotage; + let bits = sabotage::get(); + ( + block.obligation && bits & sabotage::FORGET_OBLIGATIONS == 0, + census.reached(block_idx) && bits & sabotage::FORGET_REACHED == 0, + ) + }; + if obligation + || reached + || block.data != snapshot.data + || block.end != snapshot.data.saturating_add(snapshot.offset) + { + continue; + } + skip[block_idx] = true; + any = true; + self.freed_bytes = self.freed_bytes.saturating_add(block.bytes); + if block_idx < self.resettable_general_n { + self.eden_dead_bytes = self.eden_dead_bytes.saturating_add(block.bytes); + } + self.block_skip_blocks += 1; + self.block_skip_objects = self.block_skip_objects.saturating_add(block.objects); + self.block_skip_bytes = self.block_skip_bytes.saturating_add(block.bytes); + } + #[cfg(test)] + super::super::trace::block_skip::sabotage::record_skipped_block_bases( + skip.iter() + .enumerate() + .filter(|&(_, &skipped)| skipped) + .map(|(block_idx, _)| self.block_snapshots[block_idx].data) + .collect(), + ); + if any { + #[cfg(test)] + if super::super::trace::block_skip::sabotage::get() == 0 { + verify_skipped_blocks_hold_no_live_object(&skip); + } + self.cursor.set_skip_blocks(skip); + super::super::trace::block_skip::note_block_skip_reclaimed( + self.block_skip_blocks, + self.block_skip_objects, + self.block_skip_bytes, + ); + } + } + + /// #7437: rebuild the old-gen hole free list once the object walk + /// completes — block liveness is final at that point, and the block + /// cleanup that follows only touches blocks with NO live object, which + /// the rebuild's filter already skips. + pub(super) fn push_live_block_holes(&mut self) { + if self.reclaim_dead_old_blocks { + super::old_free_rebuild_from_live_old_blocks( + &self.block_has_live, + self.old_block_start, + ); + if crate::gc::gc_diag_enabled() { + eprintln!("[gc-old-free] reusable_bytes={}", super::old_free_bytes()); + } + } + } + + pub(super) fn step(&mut self, budget: usize) -> bool { + let mut remaining = budget; + let mut done = false; + while remaining > 0 { + let Some((header_ptr, block_idx)) = self.cursor.next() else { + done = true; + break; + }; + remaining -= 1; + self.process_object(header_ptr as *mut GcHeader, block_idx); + } + // Never leave a dead header in the page index across a step boundary. + self.pending_old_unregister.flush(); + done + } + + pub(super) fn block_has_live(&self) -> &[bool] { + &self.block_has_live + } + + pub(super) fn block_snapshots(&self) -> &[crate::arena::ArenaBlockSnapshot] { + &self.block_snapshots + } + + pub(super) fn maybe_print_diag(&self) { + if !crate::gc::gc_diag_enabled() { + return; + } + let live_general = (0..self.resettable_general_n) + .filter(|&i| self.block_has_live[i]) + .count(); + let live_ll = (self.resettable_general_n..self.block_has_live.len()) + .filter(|&i| self.block_has_live[i]) + .count(); + eprintln!( + "[gc] blocks: general={} ({} live), non_general={} ({} live, survivors+longlived+old), freed_bytes={} retained_forwarded_stub_bytes={} retained_forwarded_stub_objects={} block_skip_reclaimed_blocks={} block_skip_reclaimed_objects={} block_skip_reclaimed_bytes={}", + self.resettable_general_n, + live_general, + self.block_has_live.len() - self.resettable_general_n, + live_ll, + self.freed_bytes, + self.retained_forwarded_stub_bytes, + self.retained_forwarded_stub_objects, + self.block_skip_blocks, + self.block_skip_objects, + self.block_skip_bytes, + ); + } + + fn process_object(&mut self, header: *mut GcHeader, block_idx: usize) { + unsafe { + let age_bump_this = self.do_age_bump && block_idx < self.resettable_general_n; + let flags = (*header).gc_flags; + if flags == 0 { + self.reclaim_dead_object(header, block_idx); + return; + } + if flags & GC_FLAG_PINNED != 0 { + self.keep_live_object(header, block_idx, flags, age_bump_this, true, true); + return; + } + if flags & GC_FLAG_FORWARDED != 0 { + self.process_forwarded_object(header, block_idx, flags); + return; + } + if flags & GC_FLAG_MARKED == 0 && self.unmarked_is_provably_dead(block_idx) { + self.reclaim_dead_object(header, block_idx); + } else { + self.keep_live_object(header, block_idx, flags, age_bump_this, false, true); + } + } + } + + /// Does `flags & MARKED == 0` actually prove this object is garbage? + /// + /// Only when the trace that produced the marks covered the object's + /// generation. A minor trace never marks the old generation (see + /// `minor_sweep`), so an unmarked old-gen object is merely *unvisited* — + /// it stays live and must not be finalized. #6892: reclaiming one wiped + /// the GC slot-layout mask of a live old-gen array, after which the next + /// `layout_note_slot` rebuilt the mask from a single slot and the + /// following minor stopped tracing the array's other pointer elements, + /// sweeping objects that were still referenced. + /// + /// The old-page defrag targets are exempt: this cycle evacuated every + /// indexed occupant, so the remainder is genuinely reclaimable. + #[inline] + fn unmarked_is_provably_dead(&self, block_idx: usize) -> bool { + if !self.minor_sweep || block_idx < self.old_block_start { + return true; + } + self.targeted_old_blocks + .as_ref() + .is_some_and(|selected| selected.contains(&block_idx)) + } +} + +impl ArenaSweepObjectsState { + unsafe fn keep_live_object( + &mut self, + header: *mut GcHeader, + block_idx: usize, + flags: u8, + age_bump_this: bool, + pinned: bool, + count_in_live_census: bool, + ) { + if block_idx >= self.old_block_start { + crate::arena::old_page_account_swept_object( + header as usize, + (*header).size as usize, + true, + pinned, + ); + } + if block_idx < self.block_has_live.len() { + self.block_has_live[block_idx] = true; + } + if block_idx < self.resettable_general_n { + self.eden_live_bytes = self.eden_live_bytes.saturating_add((*header).size as u64); + } + if count_in_live_census { + let size = (*header).size as u64; + self.arena_live_bytes = self.arena_live_bytes.saturating_add(size); + // #7901: the from-space share of the census, so a following copied + // minor can remove exactly what it replaces. + if crate::arena::block_in_copying_from_space( + block_idx, + self.resettable_general_n, + &self.active_survivor_blocks, + ) { + self.arena_live_from_space_bytes = + self.arena_live_from_space_bytes.saturating_add(size); + } + } + if age_bump_this && flags & GC_FLAG_TENURED == 0 { + if flags & GC_FLAG_HAS_SURVIVED != 0 { + (*header).gc_flags = + (flags | GC_FLAG_TENURED) & !GC_FLAG_HAS_SURVIVED & !GC_FLAG_MARKED; + } else { + (*header).gc_flags = (flags | GC_FLAG_HAS_SURVIVED) & !GC_FLAG_MARKED; + } + } else { + (*header).gc_flags = flags & !GC_FLAG_MARKED; + } + } + + unsafe fn process_forwarded_object( + &mut self, + header: *mut GcHeader, + block_idx: usize, + flags: u8, + ) { + // See `minor_sweep`: a minor cannot prove a stub unreferenced (old-gen + // parents are black leaves), so it must keep them all; a full trace + // reclaims the genuinely unreferenced ones. + let retain_stub = self.minor_sweep + || flags & GC_FLAG_MARKED != 0 + || (block_idx < self.resettable_general_n + && crate::arena::general_block_in_recent_window(block_idx)); + if retain_stub { + // A full collection can leave an unmarked stub in the recent-block + // safety window. It still pins the block, but it is proven dead and + // therefore excluded from live-allocation accounting. + let count_in_live_census = self.minor_sweep || flags & GC_FLAG_MARKED != 0; + self.keep_live_object(header, block_idx, flags, false, false, count_in_live_census); + if block_idx < self.resettable_general_n { + self.retained_forwarded_stub_objects = + self.retained_forwarded_stub_objects.saturating_add(1); + self.retained_forwarded_stub_bytes = self + .retained_forwarded_stub_bytes + .saturating_add((*header).size as usize); + } + return; + } + + let total_size = (*header).size as usize; + let dead_old = block_idx >= self.old_block_start; + if dead_old { + crate::arena::old_page_account_swept_object(header as usize, total_size, false, false); + } + let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE); + self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64); + layout_clear_for_ptr(user_ptr as usize); + if self.overflow_active { + gc_type_clear_dead_payload_side_tables((*header).obj_type, user_ptr as usize); + } + if self.reclaim_dead_old_blocks && dead_old { + self.pending_old_unregister.defer(header, total_size); + } else { + (*header).gc_flags = flags & !(GC_FLAG_FORWARDED | GC_FLAG_MARKED); + } + } + + unsafe fn reclaim_dead_object(&mut self, header: *mut GcHeader, block_idx: usize) { + let total_size = (*header).size as usize; + let dead_old = block_idx >= self.old_block_start; + if dead_old { + crate::arena::old_page_account_swept_object(header as usize, total_size, false, false); + } + let user_ptr = (header as *mut u8).add(GC_HEADER_SIZE); + self.freed_bytes = self.freed_bytes.saturating_add(total_size as u64); + if block_idx < self.resettable_general_n { + self.eden_dead_bytes = self.eden_dead_bytes.saturating_add(total_size as u64); + } + finalize_dead_arena_payload(header, user_ptr, self.overflow_active); + if self.reclaim_dead_old_blocks && dead_old { + self.pending_old_unregister.defer(header, total_size); + } + } +} + +/// Test builds re-check every skip against the headers themselves: a block the +/// census recorded as unreached must hold no marked or pinned object. This is +/// what turns a missed mark path into a failing unit test instead of a freed +/// live object. +#[cfg(test)] +fn verify_skipped_blocks_hold_no_live_object(skip: &[bool]) { + crate::arena::arena_walk_objects_filtered( + |block_idx| skip.get(block_idx).copied().unwrap_or(false), + |header_ptr, block_idx| unsafe { + let flags = (*(header_ptr as *const GcHeader)).gc_flags; + assert!( + flags & (GC_FLAG_MARKED | GC_FLAG_PINNED) == 0, + "block-granular sweep skipped block {block_idx}, which holds a live header {header_ptr:p} (flags {flags:#x})" + ); + }, + ); +} diff --git a/crates/perry-runtime/src/gc/tests/block_skip.rs b/crates/perry-runtime/src/gc/tests/block_skip.rs new file mode 100644 index 0000000000..ba34429dac --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/block_skip.rs @@ -0,0 +1,464 @@ +//! #10182: block-granular reclamation in the synchronous full sweep. +//! +//! Every case plants an old-generation population on a fresh thread (so the +//! arenas start empty), runs one synchronous full collection, and asks the +//! sweep which blocks it reclaimed without entering +//! (`block_skip::sabotage::last_skipped_block_bases`). Each protective +//! assertion is paired with a sabotaged run that breaks exactly the fact it +//! depends on and shows the harm the assertion exists to catch. + +use super::super::*; +use super::support::*; +use crate::gc::trace::block_skip::{block_skip_reclaimed_totals, sabotage}; + +const PLANT_BYTES: usize = 3 * crate::arena::BLOCK_SIZE + crate::arena::BLOCK_SIZE / 2; + +fn run_isolated(test: fn()) { + std::thread::spawn(move || { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + reset_global_roots(); + let _roots = ShadowAndGlobalRootResetGuard; + test(); + }) + .join() + .expect("block-skip test thread must not panic"); +} + +fn synchronous_full() { + let _ = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); +} + +/// Plant dead old-gen plain objects — strings, zero-field objects and small +/// arrays in rotation — until `bytes` are allocated. Returns every user address. +unsafe fn plant_plain(bytes: usize) -> Vec { + let mut planted = Vec::new(); + let mut allocated = 0usize; + let mut i = 0usize; + while allocated < bytes { + let user = match i % 3 { + 0 => crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_STRING) as usize, + 1 => alloc_old_test_object(0).0 as usize, + _ => alloc_old_test_array(2).0 as usize, + }; + allocated += old_test_header_and_size(user).1; + planted.push(user); + i += 1; + } + planted +} + +/// `data` of the arena block holding `user`. +fn block_base(user: usize) -> usize { + crate::arena::classify_heap_space_in_range(user) + .map(|(_, base, _)| base) + .expect("planted object must be in a registered arena block") +} + +/// Every walkable object in the block starting at `base`. +fn objects_in_block(base: usize) -> Vec { + let snapshots = crate::arena::arena_block_snapshots(); + let Some(block_idx) = snapshots.iter().position(|s| s.data == base) else { + return Vec::new(); + }; + let mut objects = Vec::new(); + crate::arena::arena_walk_objects_filtered( + |idx| idx == block_idx, + |header_ptr, _| objects.push(header_ptr as usize + GC_HEADER_SIZE), + ); + objects +} + +/// A block every walkable object of which is in `planted`, that is neither the +/// first nor the last planted block (so it is full, and not the old arena's +/// allocation block). +fn interior_planted_block(planted: &[usize]) -> (usize, Vec) { + let set: std::collections::HashSet = planted.iter().copied().collect(); + let first = block_base(planted[0]); + let last = block_base(*planted.last().unwrap()); + let mut seen = std::collections::HashSet::new(); + for &user in planted { + let base = block_base(user); + if base == first || base == last || !seen.insert(base) { + continue; + } + let objects = objects_in_block(base); + if !objects.is_empty() && objects.iter().all(|o| set.contains(o)) { + return (base, objects); + } + } + panic!("test premise: the planted population must fill an interior block"); +} + +fn skipped(base: usize) -> bool { + sabotage::last_skipped_block_bases().contains(&base) +} + +fn header_type(user: usize) -> u8 { + unsafe { (*header_from_user_ptr(user as *const u8)).obj_type } +} + +/// (a) A block holding only dead plain objects is reclaimed without the sweep +/// entering it, and the live-subject counters account every object in it. +#[test] +fn a_dead_block_of_plain_objects_is_reclaimed_without_visiting_it() { + run_isolated(|| { + let planted = unsafe { plant_plain(PLANT_BYTES) }; + let (base, objects) = interior_planted_block(&planted); + let before = block_skip_reclaimed_totals(); + + synchronous_full(); + + let after = block_skip_reclaimed_totals(); + assert!( + skipped(base), + "the all-dead interior block {base:#x} must be reclaimed without a visit" + ); + assert!(after.0 > before.0, "the skip counter must move"); + assert!( + after.1 - before.1 >= objects.len() as u64, + "every object of the skipped block must be accounted: {} < {}", + after.1 - before.1, + objects.len() + ); + assert!( + !crate::arena::pointer_in_old_gen(objects[0]), + "the skipped block must still be released by the block cleanup" + ); + }); +} + +/// (c) One reachable object keeps its whole block on the per-object path: the +/// block is not skipped, the object survives, and its dead neighbours are +/// swept one by one (an old dead header is invalidated to `obj_type == 0`). +#[test] +fn a_live_neighbour_keeps_its_block_on_the_per_object_path() { + run_isolated(|| { + let planted = unsafe { plant_plain(PLANT_BYTES) }; + let (base, objects) = interior_planted_block(&planted); + let live = *objects + .iter() + .find(|&&o| header_type(o) == GC_TYPE_OBJECT) + .expect("interior block holds a planted object"); + let dead_neighbour = *objects.iter().find(|&&o| o != live).unwrap(); + let mut root = ptr_bits(live); + js_gc_register_global_root(&mut root as *mut u64 as i64); + + synchronous_full(); + + assert!( + !skipped(base), + "a block with a reachable object must be walked" + ); + assert!( + crate::arena::pointer_in_old_gen(live), + "the reachable object's block must survive" + ); + assert_eq!(header_type(live), GC_TYPE_OBJECT); + assert_eq!( + header_type(dead_neighbour), + 0, + "the live block's dead neighbours are swept per object" + ); + assert!( + !sabotage::last_skipped_block_bases().is_empty(), + "the other all-dead blocks are still skipped in the same sweep" + ); + }); +} + +/// Sabotage for (c): with the census's reachability record broken, the same +/// population loses its reachable object's block — which is the assertion +/// above failing. +#[test] +fn sabotaged_reachability_reclaims_a_live_block() { + run_isolated(|| { + let planted = unsafe { plant_plain(PLANT_BYTES) }; + let (base, objects) = interior_planted_block(&planted); + let live = *objects + .iter() + .find(|&&o| header_type(o) == GC_TYPE_OBJECT) + .unwrap(); + let mut root = ptr_bits(live); + js_gc_register_global_root(&mut root as *mut u64 as i64); + + { + let _sabotage = sabotage::Guard::arm(sabotage::FORGET_REACHED); + synchronous_full(); + } + + assert!( + skipped(base), + "sabotage premise: the live block is skipped once reachability is forgotten" + ); + assert!( + !crate::arena::pointer_in_old_gen(live), + "and the reachable object's block is released under its root" + ); + reset_global_roots(); + }); +} + +/// Plant a population with `special` allocated in the middle of it, and +/// return `(special, block base)` with the premise that its block is interior. +unsafe fn plant_around( + special: impl FnOnce() -> T, + user_of: impl Fn(&T) -> usize, +) -> (T, usize) { + let mut planted = plant_plain(PLANT_BYTES / 2); + let value = special(); + let user = user_of(&value); + planted.push(user); + planted.extend(plant_plain(PLANT_BYTES / 2)); + let base = block_base(user); + assert_ne!(base, block_base(planted[0]), "premise: not the first block"); + assert_ne!( + base, + block_base(*planted.last().unwrap()), + "premise: not the allocation block" + ); + (value, base) +} + +/// (b) A dead promise is a finalize-hook object: its block keeps the +/// per-object path and `PromiseCleanup` drops its side-table entries. +#[test] +fn a_dead_promise_keeps_its_block_on_the_per_object_path() { + run_isolated(|| { + let (promise, base) = unsafe { + plant_around( + || { + let p = alloc_old_test_promise(); + crate::promise::scanners::test_park_promise_side_table_entries(p); + p + }, + |p| *p as usize, + ) + }; + assert_eq!( + crate::promise::scanners::test_promise_side_table_counts_for(promise as usize), + (1, 1, 1), + "premise: one entry parked in each table" + ); + + synchronous_full(); + + assert!(!skipped(base), "a block holding a promise must be walked"); + assert_eq!( + crate::promise::scanners::test_promise_side_table_counts_for(promise as usize), + (0, 0, 0), + "the dead promise's finalizer must have run" + ); + }); +} + +/// Sabotage for (b): with obligations forgotten the promise's block is skipped +/// and its finalizer never runs. +#[test] +fn sabotaged_obligations_skip_a_promise_finalizer() { + run_isolated(|| { + let (promise, base) = unsafe { + plant_around( + || { + let p = alloc_old_test_promise(); + crate::promise::scanners::test_park_promise_side_table_entries(p); + p + }, + |p| *p as usize, + ) + }; + + { + let _sabotage = sabotage::Guard::arm(sabotage::FORGET_OBLIGATIONS); + synchronous_full(); + } + + assert!( + skipped(base), + "sabotage premise: the promise block is skipped" + ); + assert_eq!( + crate::promise::scanners::test_promise_side_table_counts_for(promise as usize), + (1, 1, 1), + "and the finalizer did not run — the assertion above fails" + ); + }); +} + +/// (b) A dead Set keeps its block on the per-object path (its side allocation +/// is a finalize hook). +#[test] +fn a_dead_set_keeps_its_block_on_the_per_object_path() { + run_isolated(|| { + let ((set, elements, layout), base) = + unsafe { plant_around(|| alloc_old_test_set(4), |(s, _, _)| *s as usize) }; + let neighbour = *objects_in_block(base) + .iter() + .find(|&&o| o != set as usize) + .unwrap(); + + synchronous_full(); + + assert!(!skipped(base), "a block holding a Set must be walked"); + assert_eq!( + header_type(neighbour), + 0, + "its dead neighbours are swept per object" + ); + unsafe { retire_old_test_set(set, elements, layout) }; + }); +} + +/// A dead weak TARGET's block is walked, and the `WeakRef` clears. +/// +/// Not because the sweep owes a weak target anything — no per-object sweep +/// work is weak-specific; weak processing clears holders from mark bits before +/// the sweep. The block is walked because that processing asks the census +/// whether the target is a valid object, and a census hit records the block as +/// reached. That is conservative (it only costs the skip), so it is pinned +/// here rather than special-cased. +#[test] +fn a_dead_weak_target_block_is_walked_and_the_weak_ref_clears() { + run_isolated(|| { + let (target, base) = + unsafe { plant_around(|| alloc_old_test_object(0).0 as usize, |t| *t) }; + let holder = crate::weakref::js_weakref_new(f64::from_bits(ptr_bits(target))); + let mut root = ptr_bits(holder as usize); + js_gc_register_global_root(&mut root as *mut u64 as i64); + + synchronous_full(); + + assert!( + !skipped(base), + "weak processing's census lookup marks the target's block reached" + ); + let deref = crate::weakref::js_weakref_deref(f64::from_bits(root)); + assert_eq!( + deref.to_bits(), + crate::value::TAG_UNDEFINED, + "the WeakRef must be cleared" + ); + assert!( + !sabotage::last_skipped_block_bases().is_empty(), + "the all-dead blocks around it are still skipped" + ); + }); +} + +/// (b) An address-keyed side-table entry the dead-owner fan-out does not prune +/// — the legacy overflow table — makes object blocks obligations while it has +/// entries, and the per-object path drops the dead owner's entry. +#[test] +fn a_legacy_overflow_entry_keeps_object_blocks_on_the_per_object_path() { + run_isolated(|| { + let (owner, base) = unsafe { plant_around(|| alloc_old_test_object(0).0 as usize, |t| *t) }; + crate::state::state() + .object_hot + .overflow_fields + .borrow_mut() + .insert(owner, vec![crate::value::TAG_UNDEFINED]); + + synchronous_full(); + + assert!( + !skipped(base), + "object blocks are obligations while the table is live" + ); + assert!( + !crate::state::state() + .object_hot + .overflow_fields + .borrow() + .contains_key(&owner), + "the dead owner's overflow entry must be dropped" + ); + }); +} + +/// Sabotage for the overflow case: forgetting obligations leaves a stale entry +/// that a new object at the recycled address would inherit. +#[test] +fn sabotaged_obligations_leave_a_stale_overflow_entry() { + run_isolated(|| { + let (owner, base) = unsafe { plant_around(|| alloc_old_test_object(0).0 as usize, |t| *t) }; + crate::state::state() + .object_hot + .overflow_fields + .borrow_mut() + .insert(owner, vec![crate::value::TAG_UNDEFINED]); + + { + let _sabotage = sabotage::Guard::arm(sabotage::FORGET_OBLIGATIONS); + synchronous_full(); + } + + assert!( + skipped(base), + "sabotage premise: the owner's block is skipped" + ); + assert!( + crate::state::state() + .object_hot + .overflow_fields + .borrow() + .contains_key(&owner), + "and the stale entry survives — the assertion above fails" + ); + crate::state::state() + .object_hot + .overflow_fields + .borrow_mut() + .clear(); + }); +} + +/// (b) A side-table entry the fan-out DOES prune — an element-shape record on +/// a dead array — is dropped even though its block is skipped. +#[test] +fn a_pruned_side_table_entry_is_dropped_from_a_skipped_block() { + run_isolated(|| { + let (arr, base) = unsafe { plant_around(|| alloc_old_test_array(2).0 as usize, |a| *a) }; + crate::array::test_seed_element_shape_record(arr); + assert!(crate::array::test_element_shape_record_exists(arr)); + + synchronous_full(); + + assert!( + skipped(base), + "an element-shape record is not an obligation" + ); + assert!( + !crate::array::test_element_shape_record_exists(arr), + "the dead owner's record must be pruned by the post-trace fan-out" + ); + }); +} + +/// Only synchronous full cycles skip: a budgeted full resolves membership +/// through the page classifier and records nothing. +#[test] +fn a_budgeted_full_never_skips() { + run_isolated(|| { + let planted = unsafe { plant_plain(PLANT_BYTES) }; + let (base, _) = interior_planted_block(&planted); + sabotage::record_skipped_block_bases(vec![base]); + let before = block_skip_reclaimed_totals(); + + let mut state = + GcCycleState::new_full(GcTriggerSnapshot::capture(GcTriggerKind::OldGenBytes)); + state.set_progress_kind(GcProgressKind::NormalIncremental); + let _ = state.run_to_completion(); + + assert_eq!( + block_skip_reclaimed_totals(), + before, + "a budgeted full must not skip a block" + ); + assert!( + sabotage::last_skipped_block_bases() == vec![base], + "a budgeted sweep does not even consult the census" + ); + }); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 1a9ae7e54d..b7555bde71 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -6,6 +6,7 @@ mod barrier; mod barrier_arming; mod barrier_decoded_parent; mod block_pool_pressure; +mod block_skip; mod budgeted_step_api; mod buffer_bound_method_name; mod buffer_side_tables; diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 0232cc6a60..0901a6549c 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -1,5 +1,9 @@ use super::*; +#[path = "trace/block_skip.rs"] +pub(super) mod block_skip; +pub(super) use block_skip::BlockCensus; + crate::perry_thread_local! { /// Set by test-only helpers that wipe page metadata for isolation /// (`old_arena_page_index_clear_for_tests`): real objects become @@ -107,6 +111,15 @@ pub(crate) struct ValidPointerSet { /// full collection** (`phase_us.build_valid_pointer_set`), 12.6% of the /// `build_out` phase, and ~40 MB of transient peak heap (#7592). pub(super) arena_runs: Vec>, + /// `arena_runs[i]`'s global arena block index (`u32::MAX` for runs pushed + /// without one). The census seals a run at every block boundary, so a + /// run never straddles two blocks and a membership hit names its block + /// for free (#10182). + pub(super) arena_run_blocks: Vec, + pub(super) current_arena_run_block: u32, + /// Per-block census facts and trace reachability (#10182). Disarmed + /// unless this set was built by the production census walk. + pub(super) block_census: BlockCensus, /// `arena_runs[i].first()`, mirrored into one contiguous vector so the /// run-level binary search reads 8-byte fences instead of chasing a /// `Vec` header per probe. At 500k `json_pipeline` records this is ~4k @@ -158,6 +171,9 @@ impl ValidPointerSet { pub(super) fn new() -> Self { Self { arena_runs: Vec::new(), + arena_run_blocks: Vec::new(), + current_arena_run_block: u32::MAX, + block_census: BlockCensus::disarmed(), arena_run_firsts: Vec::new(), current_arena_run: Vec::with_capacity(VALID_POINTER_ARENA_RUN_CAPACITY), arena_count: 0, @@ -172,11 +188,17 @@ impl ValidPointerSet { /// Caller must guarantee that pushes happen in ascending address /// order — `ValidPointerSetBuilder` does so via `ArenaObjectCursor` - /// in address order. - pub(super) fn push_arena(&mut self, ptr: usize) { + /// in address order. `block_idx` is the start's global arena block; the + /// open run is sealed whenever it changes, so no run straddles two blocks + /// (#10182). + pub(super) fn push_arena_in_block(&mut self, ptr: usize, block_idx: u32) { if self.classifier_mode { return; // #6179: no exact census in classifier mode } + if block_idx != self.current_arena_run_block { + self.seal_current_arena_run(); + self.current_arena_run_block = block_idx; + } if let Some(previous) = self .current_arena_run .last() @@ -237,6 +259,7 @@ impl ValidPointerSet { // with `arena_runs` — `arena_run_firsts[i] == arena_runs[i][0]`. self.arena_run_firsts.push(sealed[0]); self.arena_runs.push(sealed); + self.arena_run_blocks.push(self.current_arena_run_block); } /// Cheap O(1) range-rejection prefilter. Most stack words and @@ -313,12 +336,13 @@ impl ValidPointerSet { /// iteration. Find the largest entry `<= query`, then validate via /// the GcHeader's size field. pub(crate) fn enclosing_object(&self, ptr: usize) -> Option { - let candidate = self.find_arena_floor(ptr)?; + let (candidate, run) = self.find_arena_floor_run(ptr)?; unsafe { let header = (candidate as *const u8).sub(GC_HEADER_SIZE) as *const GcHeader; let total = (*header).size as usize; let payload_end = candidate + total.saturating_sub(GC_HEADER_SIZE); if ptr >= candidate && ptr < payload_end { + self.note_run_reached(run); Some(candidate) } else { None @@ -326,6 +350,15 @@ impl ValidPointerSet { } } + /// A census hit in run `run`: its block was reached by the trace (#10182). + /// See `block_skip`'s module doc for why every mark passes through here. + #[inline(always)] + fn note_run_reached(&self, run: usize) { + if let Some(&block_idx) = self.arena_run_blocks.get(run) { + self.block_census.note_reached(block_idx); + } + } + /// Exact arena membership: the census runs are address-ordered, so `ptr` /// was censused iff its floor is itself. /// @@ -353,15 +386,21 @@ impl ValidPointerSet { {} censused starts are invisible to this lookup", self.current_arena_run.len() ); - self.find_arena_floor(ptr) == Some(ptr) + match self.find_arena_floor_run(ptr) { + Some((floor, run)) if floor == ptr => { + self.note_run_reached(run); + true + } + _ => false, + } } - fn find_arena_floor(&self, ptr: usize) -> Option { + fn find_arena_floor_run(&self, ptr: usize) -> Option<(usize, usize)> { let idx = self.arena_run_firsts.partition_point(|&first| first <= ptr); if idx == 0 { return None; } - Self::find_floor(&self.arena_runs[idx - 1], ptr) + Self::find_floor(&self.arena_runs[idx - 1], ptr).map(|floor| (floor, idx - 1)) } pub(super) fn find_floor(sorted: &[usize], ptr: usize) -> Option { @@ -386,6 +425,10 @@ pub(crate) fn build_valid_pointer_set() -> ValidPointerSet { pub(super) struct ValidPointerSetBuilder { set: ValidPointerSet, + /// #10182: census the per-block facts `block_skip` needs (exact census + /// only), and the block the walk is currently inside. + census_armed: bool, + census_block_idx: usize, phase: ValidPointerSetBuildPhase, arena_cursor_builder: Option, arena_cursor: Option, @@ -420,12 +463,18 @@ impl ValidPointerSetBuilder { pub(super) fn new_classifier() -> Self { let mut b = Self::new(); b.set.classifier_mode = true; + b.set.block_census = BlockCensus::disarmed(); + b.census_armed = false; b } pub(super) fn new() -> Self { + let mut set = ValidPointerSet::new(); + set.block_census = BlockCensus::armed(); Self { - set: ValidPointerSet::new(), + set, + census_armed: true, + census_block_idx: usize::MAX, phase: ValidPointerSetBuildPhase::ArenaCursorSetup, arena_cursor_builder: Some(crate::arena::ArenaObjectCursorBuilder::new( crate::arena::ArenaWalkOrder::Address, @@ -485,6 +534,7 @@ impl ValidPointerSetBuilder { if remaining == 0 { return false; } + self.set.block_census.flush_block(); self.set.finalize(); self.phase = ValidPointerSetBuildPhase::Done; return true; @@ -511,7 +561,7 @@ impl ValidPointerSetBuilder { .expect("arena cursor exists during arena walk"); cursor.next_budgeted(remaining) }; - let Some((header_ptr, _block_idx)) = next else { + let Some((header_ptr, block_idx)) = next else { let finished = self .arena_cursor .as_ref() @@ -523,14 +573,34 @@ impl ValidPointerSetBuilder { } return false; }; - self.record_arena_header(header_ptr); + if self.census_armed { + if block_idx != self.census_block_idx { + self.census_block_idx = block_idx; + if let Some((_, data, offset)) = self + .arena_cursor + .as_ref() + .and_then(crate::arena::ArenaObjectCursor::current_block_extent) + { + self.set.block_census.begin_block(block_idx, data, offset); + } + } + unsafe { + self.set + .block_census + .note_header(header_ptr as *const GcHeader); + } + } + self.record_arena_header(header_ptr, block_idx); } false } - fn record_arena_header(&mut self, header_ptr: *mut u8) { + fn record_arena_header(&mut self, header_ptr: *mut u8, block_idx: usize) { let user_ptr = unsafe { header_ptr.add(GC_HEADER_SIZE) }; - self.set.push_arena(user_ptr as usize); + self.set.push_arena_in_block( + user_ptr as usize, + u32::try_from(block_idx).unwrap_or(u32::MAX), + ); unsafe { let header = header_ptr as *const GcHeader; let flags = (*header).gc_flags; diff --git a/crates/perry-runtime/src/gc/trace/block_skip.rs b/crates/perry-runtime/src/gc/trace/block_skip.rs new file mode 100644 index 0000000000..a04e9a3b04 --- /dev/null +++ b/crates/perry-runtime/src/gc/trace/block_skip.rs @@ -0,0 +1,326 @@ +//! Block-granular reclamation for the synchronous full sweep (#10182). +//! +//! A full sweep used to visit every object in the arena: each dead one paid +//! for page accounting, `finalize_dead_arena_payload` and a page-index +//! removal, and only then did the block cleanup notice that the block had no +//! live object and reset it wholesale. After a parse/scan loop nearly every +//! block holds only dead objects, so a full's cost scaled with the garbage. +//! +//! This module lets the sweep reclaim such a block **without entering it**. +//! Two facts decide it, both gathered without any extra heap walk: +//! +//! 1. **Nothing in the block was reached by the trace.** The exact census +//! (`ValidPointerSetBuilder`) seals its address-ordered runs at block +//! boundaries and records the block each run belongs to. Every mark a +//! census-built cycle sets is preceded by a successful membership query +//! against that census (`ValidPointerSet::contains` or `enclosing_object`), +//! and a successful query marks the run's block as *reached*. A reached +//! block is a superset of a block holding a marked object, so an unreached +//! block holds none. The two mark paths that do not consult the census are +//! excluded structurally rather than recorded: allocate-black births change +//! the block's bump offset (a block whose offset moved since the census is +//! never skipped), and block persistence only force-marks the recent general +//! window, which is never skipped either. +//! +//! 2. **No object in the block owes per-object work.** The census reads every +//! header anyway; it records an *obligation* when an object is pinned, +//! forwarded, already marked, lacks `GC_FLAG_ARENA`, carries a finalize +//! hook (Map/Set side allocations, promises, native owners and views, +//! typed-array view metadata, Temporal cells, errors, RegExps, lazy tapes), +//! is an array whose raw-f64 layout bits would fire a typed-feedback +//! invalidation, or is an object while the legacy overflow table or the +//! wasm module-wrapper registry holds entries. What remains of +//! `finalize_dead_arena_payload` for the other objects is address-keyed +//! side-table removal that the full trace's dead-owner fan-out has already +//! performed at sweep entry (`ELEMENT_SHAPES`, per-object layouts, closure +//! dynamic props — `dead_owner::DEAD_KEY_PRUNES`), plus `_reserved` header +//! bits that every allocator rewrites to zero, so the block reset retires +//! them. +//! +//! The page-index and old-page bookkeeping the per-object path does for a dead +//! old object is superseded by the block cleanup, which drops every page of a +//! non-live old block (`unregister_old_block_pages`). Freed-byte and Eden-dead +//! accounting come from the census's per-block byte sums, which cover exactly +//! the objects the sweep cursor would have yielded. +//! +//! Only synchronous full cycles with a census-built pointer set use this: a +//! budgeted cycle runs the mutator between phases (births into swept holes do +//! not move an offset) and resolves membership through the page classifier, +//! so it has no census to record against. + +use super::*; +use std::cell::Cell; + +/// What the census saw in one arena block (global block index). +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct CensusBlock { + pub(crate) data: usize, + /// `data + offset` when the census walked the block. + pub(crate) end: usize, + pub(crate) objects: u64, + pub(crate) bytes: u64, + pub(crate) censused: bool, + /// Some object in the block needs the per-object sweep path. + pub(crate) obligation: bool, +} + +/// Per-block census facts and trace reachability for one cycle's +/// `ValidPointerSet`. +pub(crate) struct BlockCensus { + blocks: Vec, + reached: Vec>, + /// The block the census is inside, folded into `blocks` at the next block + /// change (`usize::MAX` when none). + current_idx: usize, + current: CensusBlock, + /// `obligation_by_type[obj_type]`: objects of this type always need the + /// per-object sweep path while this census is current. + obligation_by_type: [bool; 256], + armed: bool, +} + +impl BlockCensus { + /// An empty census that records nothing (fabricated test sets, classifier + /// mode). + pub(crate) fn disarmed() -> Self { + Self { + blocks: Vec::new(), + reached: Vec::new(), + current_idx: usize::MAX, + current: CensusBlock::default(), + obligation_by_type: [true; 256], + armed: false, + } + } + + /// A census that records blocks. The type obligations are fixed here: + /// nothing a synchronous cycle runs between the census and the sweep can + /// populate the legacy overflow table or the module-wrapper registry. + pub(crate) fn armed() -> Self { + let object_side_tables_live = !crate::object::overflow_fields_is_empty() + || crate::object::module_wrapper_registry_ever_used(); + let mut obligation_by_type = [true; 256]; + for (obj_type, slot) in obligation_by_type.iter_mut().enumerate() { + *slot = type_needs_per_object_sweep(obj_type as u8, object_side_tables_live); + } + Self { + blocks: Vec::new(), + reached: Vec::new(), + current_idx: usize::MAX, + current: CensusBlock::default(), + obligation_by_type, + armed: true, + } + } + + #[inline] + pub(crate) fn is_armed(&self) -> bool { + self.armed + } + + /// Start censusing block `block_idx` (`data`/`offset` as the cursor + /// snapshotted it). Headers noted afterwards belong to it until the next + /// `begin_block`. + #[inline] + pub(crate) fn begin_block(&mut self, block_idx: usize, data: usize, offset: usize) { + if !self.armed { + return; + } + self.flush_block(); + self.current_idx = block_idx; + self.current = CensusBlock { + data, + end: data.saturating_add(offset), + objects: 0, + bytes: 0, + censused: true, + obligation: false, + }; + } + + /// Record one censused header of the current block. Branch-light: this + /// runs once per arena object in every synchronous full's census. + /// + /// # Safety + /// `header` must be a walkable arena header inside the current block. + #[inline(always)] + pub(crate) unsafe fn note_header(&mut self, header: *const GcHeader) { + let flags = (*header).gc_flags; + let obj_type = (*header).obj_type; + let size = (*header).size as u64; + let exceptional_flags = (flags ^ GC_FLAG_ARENA) + & (GC_FLAG_ARENA | GC_FLAG_MARKED | GC_FLAG_PINNED | GC_FLAG_FORWARDED) + != 0; + let raw_f64_array = obj_type == GC_TYPE_ARRAY + && (*header)._reserved & (GC_ARRAY_RAW_F64_LAYOUT | GC_ARRAY_RAW_F64_HOLES) != 0; + let type_obligation = self.obligation_by_type[obj_type as usize]; + let block = &mut self.current; + block.objects += 1; + block.bytes += size; + block.obligation |= exceptional_flags | type_obligation | raw_f64_array; + } + + /// Fold the current block into the per-index table. Called at every block + /// change and once when the census finishes. + pub(crate) fn flush_block(&mut self) { + if !self.armed || self.current_idx == usize::MAX { + return; + } + let block_idx = self.current_idx; + if block_idx >= self.blocks.len() { + self.blocks.resize(block_idx + 1, CensusBlock::default()); + self.reached.resize_with(block_idx + 1, || Cell::new(false)); + } + self.blocks[block_idx] = self.current; + self.current_idx = usize::MAX; + } + + /// The trace reached (and so may have marked) an object in `block_idx`. + #[inline(always)] + pub(crate) fn note_reached(&self, block_idx: u32) { + if let Some(cell) = self.reached.get(block_idx as usize) { + cell.set(true); + } + } + + /// Blocks a `require_marked` whole-heap walk may skip right now: censused, + /// not reached by the trace, free of obligations (which include every + /// pinned or pre-marked header), and not grown since the census. Such a + /// block holds no marked or pinned object, so the walk would visit each + /// of its objects only to reject it. `None` when nothing qualifies or the + /// census is disarmed. + pub(crate) fn unmarked_blocks(&self) -> Option> { + if !self.armed { + return None; + } + let snapshots = crate::arena::arena_block_snapshots(); + let mut skip = vec![false; snapshots.len()]; + let mut any = false; + for (block_idx, snapshot) in snapshots.iter().enumerate() { + let Some(block) = self.block(block_idx) else { + continue; + }; + if block.obligation + || self.reached(block_idx) + || block.data != snapshot.data + || block.end != snapshot.data.saturating_add(snapshot.offset) + { + continue; + } + skip[block_idx] = true; + any = true; + } + any.then_some(skip) + } + + pub(crate) fn block(&self, block_idx: usize) -> Option { + self.blocks.get(block_idx).copied().filter(|b| b.censused) + } + + pub(crate) fn reached(&self, block_idx: usize) -> bool { + self.reached.get(block_idx).is_some_and(Cell::get) + } +} + +/// Sabotage switches for the block-skip tests: each one breaks exactly one of +/// the two facts the skip rests on, so a test can show its own assertion fails +/// when that fact is not maintained. Test builds only. +#[cfg(test)] +pub(crate) mod sabotage { + use std::cell::Cell; + + pub(crate) const FORGET_REACHED: u8 = 1; + pub(crate) const FORGET_OBLIGATIONS: u8 = 2; + + thread_local! { + static SABOTAGE: Cell = const { Cell::new(0) }; + } + + thread_local! { + static LAST_SKIPPED_BASES: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; + } + + pub(crate) fn get() -> u8 { + SABOTAGE.with(Cell::get) + } + + /// The `data` addresses of the blocks the most recent full sweep on this + /// thread reclaimed without visiting. + pub(crate) fn last_skipped_block_bases() -> Vec { + LAST_SKIPPED_BASES.with(|bases| bases.borrow().clone()) + } + + pub(crate) fn record_skipped_block_bases(bases: Vec) { + LAST_SKIPPED_BASES.with(|slot| *slot.borrow_mut() = bases); + } + + /// Arms `bits` until the guard drops. + pub(crate) struct Guard(u8); + + impl Guard { + pub(crate) fn arm(bits: u8) -> Self { + let previous = SABOTAGE.with(|s| s.replace(bits)); + Self(previous) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + SABOTAGE.with(|s| s.set(self.0)); + } + } +} + +/// Does a dead object of `obj_type` need `reclaim_dead_object`'s per-object +/// work beyond what the full trace's dead-owner fan-out and the block reset +/// already do? +pub(crate) fn type_needs_per_object_sweep(obj_type: u8, object_side_tables_live: bool) -> bool { + let Some(info) = gc_type_info(obj_type) else { + return true; + }; + if info.finalize_hook_kind != GcFinalizeHookKind::None { + return true; + } + match info.move_hook_kind { + // `clear_overflow_for_ptr` / `clear_module_wrapper_for_dead_ptr` are + // no-ops while their tables are empty; neither is in the fan-out. + GcMoveHookKind::ObjectOverflowFields => object_side_tables_live, + // Pruned post-trace by `closure::prune_dead_closure_side_table_owners`. + GcMoveHookKind::ClosureDynamicProps => false, + GcMoveHookKind::ErrorSideTables + | GcMoveHookKind::RegExpSideTables + | GcMoveHookKind::LazyArrayTape => true, + GcMoveHookKind::None + | GcMoveHookKind::MapSideTables + | GcMoveHookKind::SetSideTables + | GcMoveHookKind::ExoticExpandoOwner => false, + } +} + +crate::perry_thread_local! { + static BLOCK_SKIP_RECLAIMED_BLOCKS: Cell = const { Cell::new(0) }; + static BLOCK_SKIP_RECLAIMED_OBJECTS: Cell = const { Cell::new(0) }; + static BLOCK_SKIP_RECLAIMED_BYTES: Cell = const { Cell::new(0) }; +} + +/// Record one sweep's block-skip reclamation (live-subject counters). +pub(crate) fn note_block_skip_reclaimed(blocks: u64, objects: u64, bytes: u64) { + if blocks == 0 { + return; + } + BLOCK_SKIP_RECLAIMED_BLOCKS.with(|c| c.set(c.get().saturating_add(blocks))); + BLOCK_SKIP_RECLAIMED_OBJECTS.with(|c| c.set(c.get().saturating_add(objects))); + BLOCK_SKIP_RECLAIMED_BYTES.with(|c| c.set(c.get().saturating_add(bytes))); +} + +/// `(blocks, objects, bytes)` this thread's full sweeps reclaimed without +/// visiting, since thread start. +#[cfg(test)] +pub(crate) fn block_skip_reclaimed_totals() -> (u64, u64, u64) { + ( + BLOCK_SKIP_RECLAIMED_BLOCKS.with(Cell::get), + BLOCK_SKIP_RECLAIMED_OBJECTS.with(Cell::get), + BLOCK_SKIP_RECLAIMED_BYTES.with(Cell::get), + ) +} diff --git a/crates/perry-runtime/src/gc/verify.rs b/crates/perry-runtime/src/gc/verify.rs index cc6f9822c0..f9add564d7 100644 --- a/crates/perry-runtime/src/gc/verify.rs +++ b/crates/perry-runtime/src/gc/verify.rs @@ -521,12 +521,23 @@ pub(super) struct OldToYoungRememberedRebuildState { impl OldToYoungRememberedRebuildState { pub(super) fn new(require_marked: bool) -> Self { + Self::new_skipping(require_marked, None) + } + + /// #10182: a `require_marked` rebuild that never enters `skip`'s blocks — + /// blocks the synchronous full's census proved hold no marked or pinned + /// object (`BlockCensus::unmarked_blocks`), whose every object this walk + /// would reject anyway. + pub(super) fn new_skipping(require_marked: bool, skip: Option>) -> Self { + let mut arena_cursor = + crate::arena::ArenaObjectCursor::new(crate::arena::ArenaWalkOrder::BlockIndex); + if let Some(skip) = skip.filter(|_| require_marked) { + arena_cursor.set_skip_blocks(skip); + } Self { require_marked, sticky: StickyRememberedSet::default(), - arena_cursor: Some(crate::arena::ArenaObjectCursor::new( - crate::arena::ArenaWalkOrder::BlockIndex, - )), + arena_cursor: Some(arena_cursor), arena_done: false, malloc_index: 0, objects_scanned: 0, diff --git a/crates/perry-runtime/src/object/global_this.rs b/crates/perry-runtime/src/object/global_this.rs index 1af55af496..232391e548 100644 --- a/crates/perry-runtime/src/object/global_this.rs +++ b/crates/perry-runtime/src/object/global_this.rs @@ -21,7 +21,8 @@ use super::*; mod global_this_webassembly; pub(crate) use global_this_webassembly::{ clear_module_wrapper_for_dead_ptr, is_registered_wasm_module, module_wrapper_owner_moved, - webassembly_error_ctor_instanceof, webassembly_value_ctor_instanceof, + module_wrapper_registry_ever_used, webassembly_error_ctor_instanceof, + webassembly_value_ctor_instanceof, }; // Only the `wasm-host` engine constructs real modules (via // `webassembly::make_module_object`), so registration and trusted-handle diff --git a/crates/perry-runtime/src/object/global_this_webassembly.rs b/crates/perry-runtime/src/object/global_this_webassembly.rs index ec7def9da0..d8c0a8be01 100644 --- a/crates/perry-runtime/src/object/global_this_webassembly.rs +++ b/crates/perry-runtime/src/object/global_this_webassembly.rs @@ -240,6 +240,14 @@ pub(crate) fn module_wrapper_owner_moved(old_wrapper: usize, new_wrapper: usize) } } +/// Whether any wrapper identity was ever registered on this process. While +/// false, [`clear_module_wrapper_for_dead_ptr`] is a no-op for every address, +/// which is what lets the full sweep reclaim a dead object's whole block +/// without visiting it (#10182). +pub(crate) fn module_wrapper_registry_ever_used() -> bool { + module_wrapper_registry_used().load(std::sync::atomic::Ordering::Acquire) +} + /// Clear the identity before a dead wrapper's address can be reused. pub(crate) fn clear_module_wrapper_for_dead_ptr(wrapper: usize) { if !module_wrapper_registry_used().load(std::sync::atomic::Ordering::Acquire) { diff --git a/docs/src/internals/garbage-collector.md b/docs/src/internals/garbage-collector.md index 670d013a95..a2aea3c04f 100644 --- a/docs/src/internals/garbage-collector.md +++ b/docs/src/internals/garbage-collector.md @@ -40,6 +40,19 @@ A collection can take one of three paths: work, or `PERRY_GEN_GC=0` trace both generations and reclaim dead old objects as well as nursery garbage. +**Block-granular reclamation in the full sweep.** A synchronous full sweep +reclaims an arena block without entering it when the cycle's exact pointer +census shows that the trace reached no object in the block and that no object +in it owes per-object sweep work — no finalizer, no pinned, forwarded or +already-marked header, and no address-keyed side-table entry that the full +trace's dead-owner prune does not already drop. The block cleanup then resets +or releases the block exactly as it would after walking it. Budgeted fulls, +minors, the recent general-block window and the longlived arena keep the +per-object walk. `PERRY_GC_DIAG=1` reports the reclaimed blocks, objects and +bytes on each sweep's `[gc] blocks:` line. + + + `PERRY_GC_SCAVENGE` is on by default and lets nursery pressure route to the direct minor. `PERRY_GC_SCAVENGE_NURSERY_MB` tunes its base high-water cap, 16 MiB by default diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 1d4eba15bf..533ae24984 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -307,7 +307,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -323,7 +323,7 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "3f9e6be47b4454022b70ff2357bbdf3a80ef6a84c4986e58763acbdcce9142c1", - "crates/perry-runtime/src/gc/cycle.rs": "7ec51445743cf706fe99f089360513823b118ee13390505beeb6f4b3ba895090", + "crates/perry-runtime/src/gc/cycle.rs": "fdef083301463ad8cec8142ca86cc4354e289c67e97bbc6caea2e8d16d4bf089", "crates/perry-runtime/src/gc/mod.rs": "90339683735e4d662628cd98279a1972d523c3f2ebc358130ed3bdb0c6fc2d3f", "crates/perry-runtime/src/gc/policy.rs": "aea89274a017156efea3516b4f49124f48a66527a08195b662cfbf61672ac042", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" @@ -432,6 +432,30 @@ "verdict": "not_a_gc_pointer", "why": "#9717: monotonic count of array-growth forwarding stubs a budgeted full cycle admitted through `classifier_valid_object_start`, reported as `forwarded_stub_recoveries=` on the PERRY_GC_DIAG `[gc-incremental]` line. A `Cell` holding a tally, never an address — the stubs it counts are reached through the worklist, not retained here. Nothing for the collector." }, + { + "file": "crates/perry-runtime/src/gc/trace/block_skip.rs", + "name": "BLOCK_SKIP_RECLAIMED_BLOCKS", + "verdict": "not_a_gc_pointer", + "why": "#10182 live-subject counter: a `Cell` count of arena blocks this thread's full sweeps reclaimed without visiting. Written only by `note_block_skip_reclaimed` with a block count; holds no address." + }, + { + "file": "crates/perry-runtime/src/gc/trace/block_skip.rs", + "name": "BLOCK_SKIP_RECLAIMED_BYTES", + "verdict": "not_a_gc_pointer", + "why": "#10182 live-subject counter: a `Cell` sum of header sizes in blocks reclaimed without a visit. A byte count, never an address." + }, + { + "file": "crates/perry-runtime/src/gc/trace/block_skip.rs", + "name": "BLOCK_SKIP_RECLAIMED_OBJECTS", + "verdict": "not_a_gc_pointer", + "why": "#10182 live-subject counter: a `Cell` count of objects in blocks reclaimed without a visit. An object count, never an address." + }, + { + "file": "crates/perry-runtime/src/gc/trace/block_skip.rs", + "name": "LAST_SKIPPED_BASES", + "verdict": "test_only", + "why": "#[cfg(test)] `RefCell>` in `block_skip::sabotage`: the `data` base addresses of the arena BLOCKS the last full sweep skipped, recorded so tests can ask whether a given block was skipped. Block bases, not object pointers; never dereferenced, never traced, absent from shipped binaries." + }, { "file": "crates/perry-runtime/src/gc/verify_diag.rs", "name": "LAST_VERIFY_BUDGETED_COMPLETIONS",