From b2826346ae7517cf0436a8b5d0ee44132dac36c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 14:23:06 +0200 Subject: [PATCH 1/5] gc: a full mark-sweep promotes its Eden survivors in place; bound the promoted cohort (#10182) --- crates/perry-runtime/src/gc/cycle.rs | 187 ++++++++++++- crates/perry-runtime/src/gc/diag_sites.rs | 4 + crates/perry-runtime/src/gc/heap_budget.rs | 29 ++ crates/perry-runtime/src/gc/mod.rs | 6 +- crates/perry-runtime/src/gc/oldgen.rs | 36 +++ crates/perry-runtime/src/gc/policy.rs | 78 ++++++ .../perry-runtime/src/gc/promote_in_place.rs | 121 +++++++++ .../src/gc/tests/full_promotion.rs | 255 ++++++++++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 2 + .../src/gc/tests/promoted_cohort.rs | 56 ++++ scripts/gc_runtime_root_holders.json | 16 +- 11 files changed, 783 insertions(+), 7 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/full_promotion.rs create mode 100644 crates/perry-runtime/src/gc/tests/promoted_cohort.rs diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index a8e6eee6af..a00d50dc23 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -598,6 +598,19 @@ pub(super) struct GcCycleState { sweep: Option, freed_bytes: u64, outcome: Option, + /// #10182: may this cycle promote its young generation in place when its + /// own sweep measures it live? Decided in the constructor because the + /// SWEEP has to know (it invalidates the headers of the dead young objects + /// it reclaims so the promotion walk can tell survivors apart); the + /// decision itself is taken after the sweep, from the measurement. + promote_young_in_place: bool, + /// From-space occupancy at cycle start — the denominator of that + /// measurement. Captured in the constructor, before anything is reclaimed. + young_in_use_at_start: usize, + /// #10182: did the promotion actually run? The census `publish_reclaim_outcome` + /// hands the next copying minor depends on it — after a promotion the + /// from-space live share is zero, because there is no from-space left. + promoted_young_in_place: bool, } impl GcCycleState { @@ -639,10 +652,47 @@ impl GcCycleState { // "already traced"). Every black birth is also pushed as a mark // seed — see `gc_note_black_birth`. super::barrier::GC_BIRTH_EXTRA_FLAGS.with(|cell| cell.set(GC_FLAG_MARKED)); + let progress_kind = trigger_kind.progress_kind(GcCollectionKind::Full); + // #10182: a full mark-sweep is the one collection that can promote its + // Eden survivors for free — it has already proven them live, and its + // own sweep has already reclaimed everything around them. The guards + // are the full-cycle-specific half of the decision; the shared policy + // guards live in `full_promotion_admissible`, and the measurement that + // decides it is taken after the sweep. + // + // * **Synchronous only.** A budgeted full opens mutator windows + // between steps. The retag hands every in-use young block to + // old-gen in one operation and `reset_young_after_promotion` then + // re-seats the inline bump allocator; an allocation landing between + // the two would be an allocation into a block that is registered + // old and owned by a young arena. + // * **Generational only.** Without the generational collector there + // is no young generation to promote and no barrier to uphold. + // * **Precise roots only.** A conservative stack scan's mark set is + // not a sound liveness measurement — the same exclusion + // `seed_promote_lock_from_sweep` applies two phases later, for the + // same reason. + // * **Old→young tracking complete.** Promotion makes these objects + // old; a later minor will only find their young children through + // the remembered set. This is the predicate the copying minor's + // `BarriersInactive` fallback tests, and it means the same here. + let promote_young_in_place = !progress_kind.is_budgeted() + && super::gen_gc_enabled() + && super::full_promotion_admissible() + && matches!( + super::roots::conservative_stack_scan_decision(), + super::roots::ConservativeStackScanDecision::SkipDisabled + ) + && super::barrier::old_to_young_tracking_complete(); + let young_in_use_at_start = if promote_young_in_place { + crate::arena::copying_from_space_in_use_bytes() + } else { + 0 + }; Self { collection_kind: GcCollectionKind::Full, trigger_kind, - progress_kind: trigger_kind.progress_kind(GcCollectionKind::Full), + progress_kind, phase: GcCyclePhase::BuildValidPointerSet, trace, active_elapsed: start.elapsed(), @@ -661,6 +711,9 @@ impl GcCycleState { sweep: None, freed_bytes: 0, outcome: None, + promote_young_in_place, + young_in_use_at_start, + promoted_young_in_place: false, } } @@ -722,6 +775,9 @@ impl GcCycleState { sweep: None, freed_bytes: 0, outcome: None, + promote_young_in_place: false, + young_in_use_at_start: 0, + promoted_young_in_place: false, } } @@ -1497,7 +1553,9 @@ impl GcCycleState { .with_dead_collection_finalize( full_trace, full_trace && !self.progress_kind.is_budgeted(), - ), + ) + // #10182: see `GcCycleState::promote_young_in_place`. + .invalidating_dead_young_headers(self.promote_young_in_place), ); } let done = self @@ -1527,6 +1585,11 @@ impl GcCycleState { trace.old_pages = crate::arena::old_page_summary(); } self.sweep = Some(sweep); + // #10182: the young generation is proven and its garbage is reclaimed; + // this is the one moment a non-moving full can hand its survivors to + // old-gen for nothing. Before the remembered-set clear below, which is + // exact precisely because no young generation remains afterwards. + self.maybe_promote_young_after_full(&sweep); // #7598: seed promote-on-first-copy from THIS completed collection. // Every cycle reaching here is a full or a non-copying minor — the two // blind spots of `retune_after_scavenge`, which only copying minors @@ -1548,6 +1611,116 @@ impl GcCycleState { self.phase = GcCyclePhase::Reclaim; } + /// #10182: let a full mark-sweep promote its Eden survivors in place. + /// + /// # Why here + /// + /// A full is non-moving and promotes nothing (#7592's latch comment in + /// `gc/mod.rs`), so a parse/scan loop over document-sized inputs strands a + /// dead tree per iteration: the copying minor correctly measures the tree + /// live, promotes it untraced (#7888), and it dies one iteration later in + /// old-gen where only a full can reclaim it. The cohort bound below makes + /// that full happen; this is what stops the full from handing the NEXT + /// tree straight back to a copying minor that would evacuate all 58 MB of + /// it (survival drops as soon as two trees share Eden). + /// + /// # Ordering, and why it is this and not another + /// + /// Directly after the sweep completes, before `ReclaimSubphase::RememberedSet`: + /// + /// * **After the sweep** because the sweep is what makes the blocks + /// promotable. It has finalized and invalidated every dead young object + /// on them (`invalidating_dead_young_headers`), reset and released the + /// blocks that held nothing live, and left the survivors' page state + /// correct. Promoting before it would hand old-gen the garbage too. + /// * **Before the remembered-set clear** because that clear is + /// unconditional on a full and is *exact* only once no young generation + /// remains — which is true here by construction: `retag_young_for_in_place_promotion` + /// takes every in-use Eden and survivor block, both semispaces. + /// * **Before `publish_reclaim_outcome`**, so `finish_full_old_reclaim_baseline` + /// measures an old generation that already contains the promoted bytes. + /// That is also why nothing calls `credit_promoted_bytes_to_old_baseline` + /// here: the credit exists so a MINOR's promotion does not read as + /// old-gen growth, and a full overwrites the baseline outright moments + /// later. Crediting first would only add the bytes to the promoted + /// cohort that the same function then resets to zero — and resetting it + /// is right, because a cohort a full just traced is verified, not + /// assumed. + /// + /// # Liveness + /// + /// `PromotionLiveness::AssumeAllLive`, which after this sweep means + /// "everything still parseable", because the sweep zeroed the `obj_type` of + /// everything it reclaimed on these blocks. The cheap described page-runs + /// are therefore exact rather than approximate, and the traced path's + /// per-object header list — ~1M entries for a 58 MB tree — is not built. + fn maybe_promote_young_after_full(&mut self, sweep: &SweepTraceStats) { + if !self.promote_young_in_place { + return; + } + let young_bytes = self.young_in_use_at_start; + let live_bytes = sweep.arena_live_from_space_bytes as usize; + // The most accurate young-survival figure the collector produces: a + // full trace's own census, not a prediction. Feed it to the copying + // minor's predictor whichever way the decision goes. + super::note_full_young_survival(young_bytes, live_bytes); + if !super::full_promotion_survival_holds_up(young_bytes, live_bytes) { + super::note_full_promotion_declined(); + if crate::gc::gc_diag_enabled() { + eprintln!( + "[gc-full-promote] declined young_bytes={young_bytes} live_bytes={live_bytes} \ + survival_permille={}", + super::full_young_survival_permille(young_bytes, live_bytes), + ); + } + return; + } + let promotion = crate::arena::retag_young_for_in_place_promotion(false); + if promotion.is_empty() { + // Nothing in use to promote: the sweep reset every young block. The + // retag captured nothing, so there is nothing to undo. + super::note_full_promotion_declined(); + return; + } + let reserved_bytes = promotion.reserved_bytes(); + let blocks = promotion.block_count(); + super::note_promoted_young_capacity(reserved_bytes); + let stats = crate::arena::finish_in_place_promotion( + promotion, + crate::arena::PromotionLiveness::AssumeAllLive, + ); + self.promoted_young_in_place = true; + super::note_in_place_promotion(stats.bytes, stats.live_bytes, stats.objects); + super::note_full_promotion(stats.bytes, stats.objects); + // Deliberately NOT `instruments::note_copying_minor_moved`: that bumps + // `copying_minor_cycles`, and a full counted as a copying minor is + // #7025's shape — a liveness counter summing two collectors, so a cell + // can pass having run no copying minor at all. + if let Some(trace) = self.trace.as_mut() { + trace.old_pages = crate::arena::old_page_summary(); + } + if crate::gc::gc_diag_enabled() { + eprintln!( + "[gc-full-promote] promoted blocks={blocks} objects={} bytes={} \ + reserved_bytes={reserved_bytes} young_bytes={young_bytes} \ + live_bytes={live_bytes} survival_permille={} cycles={} declined={}", + stats.objects, + stats.bytes, + super::full_young_survival_permille(young_bytes, live_bytes), + super::full_promotion_cycles(), + super::full_promotion_declined_cycles(), + ); + } + debug_assert_eq!( + crate::arena::copying_from_space_in_use_bytes(), + 0, + "a promoting full must leave the young generation EMPTY: the \ + remembered-set clear below is exact only if nothing young remains, \ + and the next copying minor would otherwise evacuate a nursery that \ + still holds the tree this full just proved live" + ); + } + fn step_reclaim(&mut self, budget: GcWorkBudget) { self.reclaim_state .get_or_insert_with(ReclaimCycleState::new); @@ -1750,7 +1923,15 @@ impl GcCycleState { let (arena_live_bytes, from_space_live) = match self.sweep { Some(sweep) => ( sweep.arena_live_bytes as usize, - Some(sweep.arena_live_from_space_bytes as usize), + // #10182: a promoting full leaves NO from-space. Publishing the + // sweep's from-space share here would hand the next copied + // minor bytes to subtract that are no longer in from-space at + // all — the #7901 double-charge, one collection removed. + Some(if self.promoted_young_in_place { + 0 + } else { + sweep.arena_live_from_space_bytes as usize + }), ), None => (crate::arena::arena_live_allocated_bytes(), None), }; diff --git a/crates/perry-runtime/src/gc/diag_sites.rs b/crates/perry-runtime/src/gc/diag_sites.rs index 5d7776bfab..88069f811b 100644 --- a/crates/perry-runtime/src/gc/diag_sites.rs +++ b/crates/perry-runtime/src/gc/diag_sites.rs @@ -54,11 +54,15 @@ pub(super) fn trigger_decision(site: &'static str, kind: &'static str) { let next_malloc = policy::GC_NEXT_MALLOC_TRIGGER.with(Cell::get); let old_in_use = crate::arena::old_gen_in_use_bytes(); let old_free = old_free_bytes(); + let promoted_since_full = policy::promoted_bytes_since_full(); + let cohort_bound = + policy::promoted_cohort_bound_bytes(policy::GC_OLD_LIVE_AT_LAST_FULL.with(Cell::get)); eprintln!( "[gc-trigger] site={site} kind={kind} arena_total={arena_total} next_base={next_base} armed={armed} \ from_space={from_space} nursery_cap={nursery_cap} old_in_use={old_in_use} old_free={old_free} \ old_reclaimable={old_reclaimable} external_side={external} old_baseline={old_baseline} \ old_band={old_band} old_threshold={old_threshold} old_pending={old_pending} retaining={retaining} \ + promoted_since_full={promoted_since_full} cohort_bound={cohort_bound} \ malloc={malloc} next_malloc={next_malloc}" ); } diff --git a/crates/perry-runtime/src/gc/heap_budget.rs b/crates/perry-runtime/src/gc/heap_budget.rs index 9c58739038..56c0f71439 100644 --- a/crates/perry-runtime/src/gc/heap_budget.rs +++ b/crates/perry-runtime/src/gc/heap_budget.rs @@ -164,6 +164,35 @@ budget_scaled_accessor!( 12, 2 * 1024 * 1024 ); +/// #10182: floor of the promoted-but-unverified old-gen cohort — **one base +/// nursery cap**, budget-scaled like every other threshold in this file. +/// +/// Not a constant of its own, and not the 64 MB the first draft used. The +/// cohort this bounds is produced one nursery at a time: every in-place +/// promotion hands the whole young generation to old-gen, so the quantum of +/// the thing being bounded IS the nursery cap. A floor of one quantum is the +/// tightest bound that cannot fire twice for a single promotion, and it is +/// what makes a parse/scan loop over document-sized inputs reach the +/// fulls-dominated regime instead of stranding a dead tree per iteration +/// (#10182). At 64 MB — four quanta on the 16 MB default cap — the arm fired +/// only after the third or fourth promotion, i.e. after the peak it was meant +/// to cap had already been set: measured 200/276/210 MiB against main's +/// 188/256/187 on the three target rows. +/// +/// The denominator tracks `gc_scavenge_nursery_cap_bytes`'s own default, so a +/// budget-constrained device gets a floor in the same proportion to its heap +/// that a desktop gets to the 16 MB cap. +pub(crate) fn gc_promoted_cohort_floor_dyn_bytes() -> usize { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + budget_scaled( + super::policy::gc_scavenge_nursery_cap_bytes(), + 1, + 24, + 1024 * 1024, + ) + }) +} budget_scaled_accessor!( gc_copy_promotion_handoff_min_dyn_bytes, GC_COPY_PROMOTION_HANDOFF_MIN_BYTES, diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 9223e1c058..62484e5505 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -232,8 +232,10 @@ mod native_stack_scan; mod promote_in_place; use promote_in_place::*; pub use promote_in_place::{ - first_cycle_promotion_attempts, first_cycle_promotion_rollbacks, in_place_promoted_objects, - in_place_promotion_cycles, untraced_promoted_objects, untraced_promotion_cycles, + first_cycle_promotion_attempts, first_cycle_promotion_rollbacks, full_promoted_bytes, + full_promoted_objects, full_promotion_cycles, full_promotion_declined_cycles, + in_place_promoted_objects, in_place_promotion_cycles, untraced_promoted_objects, + untraced_promotion_cycles, }; /// Instrument-liveness counters (#7604): copying minors completed, objects /// relocated, loop back-edge polls reached. Mode-independent — they count what diff --git a/crates/perry-runtime/src/gc/oldgen.rs b/crates/perry-runtime/src/gc/oldgen.rs index 0a96aea979..1f6d7b026c 100644 --- a/crates/perry-runtime/src/gc/oldgen.rs +++ b/crates/perry-runtime/src/gc/oldgen.rs @@ -1240,6 +1240,25 @@ impl IncrementalSweepState { self } + /// #10182: a synchronous full that intends to promote its young generation + /// in place must leave the young blocks in a state the promotion walk can + /// read. `finish_in_place_promotion`'s only liveness sources are the mark + /// bits (which this sweep clears as it goes) and "is this header's + /// `obj_type` arena-walkable" — so the sweep invalidates the headers of the + /// dead young objects it reclaims, exactly the way it already invalidates + /// dead OLD ones (`invalidate_dead_old_arena_header`). After that the + /// promotion's `PromotionLiveness::AssumeAllLive` walk registers precisely + /// the survivors, and the page-run description re-parses to the same count + /// because producer and expander apply the same `gc_type_is_arena_walkable` + /// filter. + /// + /// No `unregister_old_object_pages` here: these objects were young, so they + /// were never in the old-gen page index to be removed from. + pub(super) fn invalidating_dead_young_headers(mut self, on: bool) -> Self { + self.arena.invalidate_dead_young_headers = on; + self + } + pub(super) fn step(&mut self, budget: usize) -> bool { match self.subphase { SweepCycleSubphase::CollectionSideBuffers => { @@ -1392,6 +1411,8 @@ struct ArenaSweepObjectsState { /// #7901: see `SweepTraceStats::arena_live_from_space_bytes`. arena_live_from_space_bytes: u64, active_survivor_blocks: std::ops::Range, + /// #10182: see `IncrementalSweepState::invalidating_dead_young_headers`. + invalidate_dead_young_headers: bool, } impl ArenaSweepObjectsState { @@ -1427,6 +1448,7 @@ impl ArenaSweepObjectsState { arena_live_bytes: 0, arena_live_from_space_bytes: 0, active_survivor_blocks: crate::arena::active_survivor_block_index_range(), + invalidate_dead_young_headers: false, } } @@ -1651,6 +1673,20 @@ impl ArenaSweepObjectsState { finalize_dead_arena_payload(header, user_ptr, self.overflow_active); if self.reclaim_dead_old_blocks && dead_old { self.pending_old_unregister.defer(header, total_size); + } else if self.invalidate_dead_young_headers + && crate::arena::block_in_copying_from_space( + block_idx, + self.resettable_general_n, + &self.active_survivor_blocks, + ) + { + // #10182: this block is about to be handed to old-gen whole. Leave + // the header unparseable as an object so the promotion walk and the + // page-run expansion both skip it. `size` is deliberately preserved + // — every arena walker hops by it, this one included. + (*header).obj_type = 0; + (*header).gc_flags = 0; + (*header)._reserved = 0; } } } diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index b2a25b101c..99e14eabec 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1085,6 +1085,13 @@ crate::perry_thread_local! { pub(super) static GC_DEFERRED_REQUEST: Cell = const { Cell::new(DeferredGcRequest::None) }; pub(super) static GC_OLD_RECLAIM_PENDING: Cell = const { Cell::new(false) }; + /// #10182: bytes promoted into old-gen since the last full old reclaim. + /// Every promotion is credited to the old baseline so it never reads as + /// growth; this is the running total of that credited, unverified cohort. + pub(super) static GC_PROMOTED_SINCE_FULL: Cell = const { Cell::new(0) }; + /// #10182: old-gen occupancy the last full old reclaim left behind — the + /// verified live set the cohort bound is proportional to. + pub(super) static GC_OLD_LIVE_AT_LAST_FULL: Cell = const { Cell::new(0) }; pub(super) static GC_LAST_OLD_RECLAIM_IN_USE_BYTES: Cell = const { Cell::new(0) }; /// Live allocated arena bytes measured right after the last FULL /// mark-sweep — the baseline for major-GC pacing @@ -1355,6 +1362,9 @@ impl Drop for OldReclaimReentryGuard { pub(super) const GC_OLD_GEN_RECLAIM_THRESHOLD_BYTES: usize = 48 * 1024 * 1024; pub(super) const GC_OLD_GEN_RECLAIM_GROWTH_BYTES: usize = 32 * 1024 * 1024; +/// #10182: the cohort bound above the floor, as a multiple of the old-gen +/// live set the last full verified. +const PROMOTED_COHORT_GROWTH_MULTIPLIER: usize = 2; pub(super) const GC_COPY_PROMOTION_HANDOFF_MIN_BYTES: usize = 24 * 1024 * 1024; #[inline] @@ -1737,6 +1747,7 @@ pub(super) fn old_reclaim_pressure_due(old_in_use: usize, baseline: usize) -> bo && !GC_MAJOR_PACING_RETAINING.with(|c| c.get()); crossed_absolute_threshold || old_in_use.saturating_sub(baseline) >= gc_old_reclaim_growth_band_bytes(baseline) + || promoted_cohort_bound_due() } /// Whether an imminent promotion justifies a full old reclaim FIRST. @@ -1922,6 +1933,62 @@ pub(super) fn credit_promoted_bytes_to_old_baseline(promoted_bytes: usize) { } GC_LAST_OLD_RECLAIM_IN_USE_BYTES .with(|bytes| bytes.set(bytes.get().saturating_add(promoted_bytes))); + // #10182: the credit makes promoted bytes invisible to the growth band by + // design; the cohort bound is what still sees them. + GC_PROMOTED_SINCE_FULL.with(|bytes| bytes.set(bytes.get().saturating_add(promoted_bytes))); +} + +/// #10182: how many promoted-but-unverified bytes old-reclaim may leave +/// unexamined before a full is due: `max(floor, k × old live at last full)`. +/// +/// The growth band cannot see this cohort: promotion credits the old baseline +/// (#7965, and rightly — a pinned baseline degenerates the band), so a loop +/// that parses a document, promotes it whole (untraced, #7888) and drops it +/// leaves one dead tree per iteration in old-gen that `old_in_use − baseline` +/// never counts, the absolute first-crossing arm is exempted while the heap is +/// classified retaining, and the untraced-cohort instruments only act on a +/// contradicting survival measurement the workload never produces (the tree +/// IS live at every minor; it dies afterwards). Measured on +/// `records_array_8m:scan`: 4 minors, 0 fulls, 105 MB of old-gen holding one +/// live 23 MB tree, peak RSS 188 MiB against Node's 110. +/// +/// Bounding the unverified cohort by a multiple of the verified live set keeps +/// total major work linear in the live set (the #7592 argument, applied to +/// promotions instead of arena growth) while capping residency: a full every +/// `k × live` bytes of promotion, at a cost proportional to `live`. +/// +/// The floor is ONE base nursery cap (`gc_promoted_cohort_floor_dyn_bytes`, +/// which carries the argument), not a separate constant. +/// +/// ★ This arm is paired with a full that promotes its own Eden survivors in +/// place (`GcCycleState::maybe_promote_young_after_full`). A non-promoting full +/// leaves the tree it just proved live in Eden, where the next copying minor +/// can find it sharing the nursery with the following tree and evacuate it. +/// The first draft of this bound shipped without that pairing, with a 64 MB +/// floor, and measured peak RSS UP on all three target rows of #10182. +pub(super) fn promoted_cohort_bound_bytes(old_live_at_last_full: usize) -> usize { + super::gc_promoted_cohort_floor_dyn_bytes() + .max(old_live_at_last_full.saturating_mul(PROMOTED_COHORT_GROWTH_MULTIPLIER)) +} + +#[inline] +fn promoted_cohort_bound_due() -> bool { + GC_PROMOTED_SINCE_FULL.with(Cell::get) + >= promoted_cohort_bound_bytes(GC_OLD_LIVE_AT_LAST_FULL.with(Cell::get)) +} + +/// Trace/test observability for the cohort bound. +pub(crate) fn promoted_bytes_since_full() -> usize { + GC_PROMOTED_SINCE_FULL.with(Cell::get) +} + +#[cfg(test)] +pub(super) fn seed_promoted_cohort_for_tests( + promoted_since_full: usize, + old_live_at_last_full: usize, +) { + GC_PROMOTED_SINCE_FULL.with(|c| c.set(promoted_since_full)); + GC_OLD_LIVE_AT_LAST_FULL.with(|c| c.set(old_live_at_last_full)); } /// Feed a copying minor's measured young-survival ratio to arena-growth pacing. @@ -1999,6 +2066,9 @@ pub(super) fn finish_full_old_reclaim_baseline() { let old_in_use = old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|bytes| bytes.set(old_in_use)); + // #10182: this full verified everything old; the cohort starts over. + GC_OLD_LIVE_AT_LAST_FULL.with(|bytes| bytes.set(old_in_use)); + GC_PROMOTED_SINCE_FULL.with(|bytes| bytes.set(0)); // Record the TOTAL post-full live set for major-GC pacing (young+old): the // full sweep is the only collection that frees forwarding stubs, so this is // the "clean" size the arena returns to and the base for the K× growth gate. @@ -3212,6 +3282,14 @@ pub(crate) fn trigger_path_hot_slot_indices() -> Vec<(&'static str, u32)> { "GC_MAJOR_PACING_RETAINING", GC_MAJOR_PACING_RETAINING.slot_index(), ), + ( + "GC_PROMOTED_SINCE_FULL", + GC_PROMOTED_SINCE_FULL.slot_index(), + ), + ( + "GC_OLD_LIVE_AT_LAST_FULL", + GC_OLD_LIVE_AT_LAST_FULL.slot_index(), + ), ("GC_FLAGS", GC_FLAGS.slot_index()), ( "GC_BUDGETED_CYCLE_ACTIVE", diff --git a/crates/perry-runtime/src/gc/promote_in_place.rs b/crates/perry-runtime/src/gc/promote_in_place.rs index 9e36e15ad0..7558ae595f 100644 --- a/crates/perry-runtime/src/gc/promote_in_place.rs +++ b/crates/perry-runtime/src/gc/promote_in_place.rs @@ -192,6 +192,15 @@ thread_local! { /// rollback. static FIRST_CYCLE_PROMOTION_ATTEMPTS: Cell = const { Cell::new(0) }; static FIRST_CYCLE_PROMOTION_ROLLBACKS: Cell = const { Cell::new(0) }; + /// #10182 live-subject counters for the promotion a FULL mark-sweep + /// performs. Separate from `IN_PLACE_PROMOTION_CYCLES` (which they also + /// feed) because "a full promoted its Eden survivors" is the claim every + /// acceptance measurement of #10182 rests on, and a counter that also + /// counts copying minors cannot carry it. + static FULL_PROMOTION_CYCLES: Cell = const { Cell::new(0) }; + static FULL_PROMOTION_DECLINED_CYCLES: Cell = const { Cell::new(0) }; + static FULL_PROMOTED_OBJECTS: Cell = const { Cell::new(0) }; + static FULL_PROMOTED_BYTES: Cell = const { Cell::new(0) }; } /// Record that `bytes` of young capacity became old-gen, so the next @@ -370,6 +379,118 @@ pub(super) fn should_promote_young_in_place() -> bool { .is_some_and(|permille| permille >= PROMOTE_SURVIVAL_THRESHOLD_PERMILLE) } +/// May a synchronous FULL mark-sweep promote its young generation in place? +/// (#10182) +/// +/// This answers only "may it try", the way +/// [`should_attempt_first_cycle_promotion`] does — and for the same reason. +/// The copying minor has to PREDICT the ratio (it decides before its trace); +/// a full decides after its own trace and its own sweep, so it has the +/// measurement itself. [`full_promotion_survival_holds_up`] is the decision. +/// +/// The shared guards are exactly `in_place_promotion_admissible`'s: the +/// `PERRY_GC_PROMOTE_IN_PLACE` kill switch, the forced-evacuation stress modes +/// (which exist to make objects move — leave them a copier to drive), and the +/// running dead-byte budget. The caller adds the ones that are specific to a +/// full cycle: it must be synchronous (a budgeted full opens mutator windows +/// between steps, and the retag is not an operation that can be interleaved +/// with allocation), generational, and its stack roots must be precise — a +/// conservative scan's mark set is not a sound liveness measurement, which is +/// the same reason `seed_promote_lock_from_sweep` excludes those cycles. +pub(super) fn full_promotion_admissible() -> bool { + in_place_promotion_admissible() +} + +/// Did the full's own sweep measure the young generation as live enough to +/// promote it whole? (#10182) +/// +/// `young_bytes` is from-space occupancy captured at cycle start, +/// `live_bytes` the from-space share of the sweep's live census. The threshold +/// is [`PROMOTE_SURVIVAL_THRESHOLD_PERMILLE`], the same 95% the copying minor +/// uses, and it means the same thing: at most 5% of the promoted bytes are +/// dead, and they cost footprint until the next full. +/// +/// ★ The number this tests is a MEASUREMENT, not a prediction. The copying +/// minor reads the *previous* cycle's ratio because it must decide before it +/// traces; this reads the ratio this very cycle just measured. A misprediction +/// is therefore impossible here — what remains is the ordinary policy question +/// of whether a 95%-live nursery is worth tenuring whole, and the answer is +/// the one #7742 already argued. +pub(super) fn full_promotion_survival_holds_up(young_bytes: usize, live_bytes: usize) -> bool { + if young_bytes == 0 { + return false; + } + full_young_survival_permille(young_bytes, live_bytes) >= PROMOTE_SURVIVAL_THRESHOLD_PERMILLE +} + +/// The ratio itself, in permille, clamped to 1000. Shared by the decision and +/// the diagnostic so the two cannot disagree. +pub(super) fn full_young_survival_permille(young_bytes: usize, live_bytes: usize) -> u64 { + (live_bytes as u64) + .saturating_mul(1000) + .checked_div(young_bytes as u64) + .unwrap_or(0) + .min(1000) +} + +/// Record a full's exact young-survival measurement for the copying minor's +/// predictor (#10182). +/// +/// Deliberately NOT [`note_young_survival`]. That function does three other +/// things — it settles the untraced run, re-bases the untraced budget against +/// the current old-gen, and can raise an old-reclaim request — all of which a +/// full either does itself or makes meaningless: `finish_full_old_reclaim_baseline` +/// runs `note_full_collection_reclaimed_old_gen` moments later and resets both +/// the untraced run and the budget base, and clears any old-reclaim request. +/// Calling it would therefore be a no-op wrapped around a redundant request, +/// and the redundant request is the kind that schedules a second full behind +/// the one that just ran. +/// +/// What IS worth keeping is the ratio: it is the most accurate young-survival +/// figure the collector ever produces, and the next copying minor's in-place +/// decision reads exactly this cell. +pub(super) fn note_full_young_survival(young_bytes: usize, live_bytes: usize) { + if young_bytes == 0 { + return; + } + LAST_YOUNG_SURVIVAL_PERMILLE + .with(|c| c.set(Some(full_young_survival_permille(young_bytes, live_bytes)))); +} + +/// A full mark-sweep promoted its Eden survivors in place (#10182). +pub(super) fn note_full_promotion(promoted_bytes: usize, promoted_objects: usize) { + FULL_PROMOTION_CYCLES.with(|c| c.set(c.get().saturating_add(1))); + FULL_PROMOTED_OBJECTS.with(|c| c.set(c.get().saturating_add(promoted_objects as u64))); + FULL_PROMOTED_BYTES.with(|c| c.set(c.get().saturating_add(promoted_bytes))); +} + +/// A full mark-sweep was ELIGIBLE to promote and its own measurement said no. +/// +/// The counterpart counter matters as much as the positive one: "the promotion +/// never fired" and "the promotion was never eligible" are different verdicts, +/// and only one of them is a policy result. +pub(super) fn note_full_promotion_declined() { + FULL_PROMOTION_DECLINED_CYCLES.with(|c| c.set(c.get().saturating_add(1))); +} + +/// Live-subject counters for the full-collection promotion (#10182): a green +/// acceptance run that never promoted at a full proves nothing about it. +pub fn full_promotion_cycles() -> u64 { + FULL_PROMOTION_CYCLES.with(Cell::get) +} + +pub fn full_promotion_declined_cycles() -> u64 { + FULL_PROMOTION_DECLINED_CYCLES.with(Cell::get) +} + +pub fn full_promoted_objects() -> u64 { + FULL_PROMOTED_OBJECTS.with(Cell::get) +} + +pub fn full_promoted_bytes() -> usize { + FULL_PROMOTED_BYTES.with(Cell::get) +} + /// The untraced-promotion budget, and therefore **the worst-case retained /// garbage of the untraced path**: every byte it admits is *assumed* live, so /// after an abrupt live→dead phase change every one of them can be garbage diff --git a/crates/perry-runtime/src/gc/tests/full_promotion.rs b/crates/perry-runtime/src/gc/tests/full_promotion.rs new file mode 100644 index 0000000000..7226b0f6be --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/full_promotion.rs @@ -0,0 +1,255 @@ +//! #10182: a synchronous full mark-sweep promotes its Eden survivors in place. +//! +//! What these pin, in order of how badly a regression would bite: +//! +//! 1. **After a promoting full the young generation is EMPTY** and the next +//! copying minor moves nothing of the tree the full proved live. The failure +//! this exists to prevent is the one the cohort-bound experiment on #10182 +//! measured: a full that leaves a 58 MB tree in Eden, followed by a minor +//! whose survival reads ~50% (the previous tree is still there), is an +//! evacuating copy of the whole tree — the worst outcome on the table. +//! 2. **The promoted objects are first-class old-gen objects**: same address, +//! `GC_FLAG_TENURED`, and indexed on their old page. An unindexed promoted +//! object is a missed old→young edge one minor later. +//! 3. **The dead ones are NOT indexed.** The sweep invalidates their headers so +//! the described page runs re-parse to exactly the survivors. Asserted by +//! walking the index, because the `debug_assert` inside the run expansion is +//! compiled out of the `--release` test build this suite runs under. +//! 4. **Both states of the decision**: a mostly-dead nursery declines (and says +//! so in its own counter), and a thread that is not admitted never plans. +//! +//! Each test runs on a fresh thread: the arenas are thread-local, so that is +//! what makes "survival" a property of the heap the test built rather than of +//! whatever an earlier test left in Eden. + +use super::super::promote_in_place::InPlacePromotionTestGuard; +use super::super::*; +use super::support::*; + +const LIVE_LEAVES: u32 = 400; + +/// Empty the young generation with one promoting copying minor, so the full +/// that follows measures only what the test allocates next. +fn empty_the_young_generation() { + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert_copied_minor_trace(&trace, true, CopiedMinorFallbackReason::None, false); + assert_eq!( + crate::arena::copying_from_space_in_use_bytes(), + 0, + "precondition: the clearing minor promoted in place and left no young generation" + ); +} + +fn full_collect() { + gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Manual)); +} + +fn user_to_header(user: usize) -> usize { + unsafe { header_from_user_ptr(user as *const u8) as usize } +} + +fn indexed_headers_on_pages_of(addrs: &[usize]) -> std::collections::BTreeSet { + let mut pages = crate::fast_hash::new_ptr_hash_set(); + for &addr in addrs { + pages.insert(crate::arena::generation_page_for_addr(addr)); + } + let mut seen = std::collections::BTreeSet::new(); + crate::arena::old_arena_walk_objects_on_pages(&pages, |h| { + seen.insert(h as usize); + }); + seen +} + +#[test] +fn a_full_over_a_live_nursery_promotes_it_in_place_and_leaves_no_young_generation() { + std::thread::spawn(|| { + let _copying = CopyingNurseryTestGuard::new(4); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _precise = ConservativeScanDisabledGuard::new(); + let _promote = InPlacePromotionTestGuard::enabled(1000); + empty_the_young_generation(); + + // One live tree: an array sized up front (no growth, so no abandoned + // backing store and no forwarding stub) holding LIVE_LEAVES young + // strings, plus ONE unrooted leaf in the middle of it — survival stays + // well above 95%, and the dead leaf is the object assertion 3 needs. + let mut arr = crate::array::js_array_alloc(LIVE_LEAVES); + let mut leaves = Vec::with_capacity(LIVE_LEAVES as usize); + let mut dead = 0usize; + for i in 0..LIVE_LEAVES { + if i == LIVE_LEAVES / 2 { + dead = young_leaf(); + } + let leaf = young_leaf(); + leaves.push(leaf); + arr = crate::array::js_array_push_f64(arr, f64::from_bits(string_bits(leaf))); + } + let arr = arr as usize; + js_shadow_slot_set(0, ptr_bits(arr)); + assert!(crate::arena::pointer_in_nursery(arr)); + assert!(crate::arena::pointer_in_nursery(dead)); + + let cycles_before = crate::gc::full_promotion_cycles(); + let declined_before = crate::gc::full_promotion_declined_cycles(); + let objects_before = crate::gc::full_promoted_objects(); + full_collect(); + + // Live subject first: a green run that never promoted proves nothing. + assert_eq!( + crate::gc::full_promotion_cycles() - cycles_before, + 1, + "the full must have promoted its young generation (declined {} times)", + crate::gc::full_promotion_declined_cycles() - declined_before + ); + let promoted = crate::gc::full_promoted_objects() - objects_before; + assert!( + promoted > LIVE_LEAVES as u64, + "the promotion must have covered the array and all {LIVE_LEAVES} leaves, got {promoted}" + ); + assert!( + super::super::promote_in_place::last_young_survival_permille() + .is_some_and(|p| p >= 950), + "the full's own measurement must have been fed to the predictor" + ); + + // (1) No young generation remains. + assert_eq!( + crate::arena::copying_from_space_in_use_bytes(), + 0, + "a promoting full must leave the young generation empty" + ); + + // (2) Same address, old-gen, TENURED, indexed. + assert_eq!((js_shadow_slot_get(0) & POINTER_MASK) as usize, arr); + for &addr in std::iter::once(&arr).chain(leaves.iter()) { + assert!( + crate::arena::pointer_in_old_gen(addr), + "{addr:#x} must classify as old-gen after the promotion" + ); + let header = user_to_header(addr) as *const GcHeader; + assert_ne!( + unsafe { (*header).gc_flags } & GC_FLAG_TENURED, + 0, + "a promoted object must carry GC_FLAG_TENURED (#7511)" + ); + } + let mut addrs = vec![arr]; + addrs.extend_from_slice(&leaves); + addrs.push(dead); + let indexed = indexed_headers_on_pages_of(&addrs); + assert!( + indexed.contains(&user_to_header(arr)), + "the promoted array must be indexed on its old page" + ); + for &leaf in &leaves { + assert!( + indexed.contains(&user_to_header(leaf)), + "every promoted leaf must be indexed on its old page" + ); + } + + // (3) The dead leaf was reclaimed by the sweep, invalidated, and is + // not in the index a dirty-page scan would walk. + let dead_header = user_to_header(dead); + assert_eq!( + unsafe { (*(dead_header as *const GcHeader)).obj_type }, + 0, + "the sweep must have invalidated the dead young header before the promotion" + ); + assert!( + !indexed.contains(&dead_header), + "a dead object on a promoted block must not be indexed as live" + ); + + // The data itself is intact. + for (i, &leaf) in leaves.iter().enumerate() { + let bits = crate::array::js_array_get_f64(arr as *const _, i as u32).to_bits(); + assert_eq!((bits & POINTER_MASK) as usize, leaf); + } + + // (1, continued) The next copying minor has nothing to move. + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert_eq!( + trace.copying_nursery.copied_objects, 0, + "a minor after a promoting full must not evacuate the promoted tree" + ); + assert_eq!((js_shadow_slot_get(0) & POINTER_MASK) as usize, arr); + assert!(crate::arena::pointer_in_old_gen(arr)); + }) + .join() + .expect("full-promotion test thread must not panic"); +} + +#[test] +fn a_full_over_a_mostly_dead_nursery_declines_and_keeps_its_young_generation() { + std::thread::spawn(|| { + let _copying = CopyingNurseryTestGuard::new(4); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _precise = ConservativeScanDisabledGuard::new(); + let _promote = InPlacePromotionTestGuard::enabled(1000); + empty_the_young_generation(); + + for _ in 0..2_000 { + std::hint::black_box(young_leaf()); + } + let survivor = young_leaf(); + js_shadow_slot_set(0, string_bits(survivor)); + + let cycles_before = crate::gc::full_promotion_cycles(); + let declined_before = crate::gc::full_promotion_declined_cycles(); + full_collect(); + + assert_eq!( + crate::gc::full_promotion_declined_cycles() - declined_before, + 1, + "the full was admitted, measured a mostly-dead nursery, and must say it declined" + ); + assert_eq!(crate::gc::full_promotion_cycles(), cycles_before); + assert!( + super::super::promote_in_place::last_young_survival_permille() + .is_some_and(|p| p < 950), + "the declining measurement must still reach the copying minor's predictor" + ); + assert!( + crate::arena::pointer_in_nursery(survivor), + "a declined promotion must leave the survivor young" + ); + }) + .join() + .expect("full-promotion test thread must not panic"); +} + +#[test] +fn a_full_on_a_thread_not_admitted_to_in_place_promotion_never_plans_one() { + std::thread::spawn(|| { + let _copying = CopyingNurseryTestGuard::new(4); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _precise = ConservativeScanDisabledGuard::new(); + // No `InPlacePromotionTestGuard`: `in_place_promotion_admissible` is + // false, which is also the state `PERRY_GC_PROMOTE_IN_PLACE=0` produces. + let survivor = young_leaf(); + js_shadow_slot_set(0, string_bits(survivor)); + let dead = young_leaf(); + + let cycles_before = crate::gc::full_promotion_cycles(); + let declined_before = crate::gc::full_promotion_declined_cycles(); + full_collect(); + + assert_eq!(crate::gc::full_promotion_cycles(), cycles_before); + assert_eq!( + crate::gc::full_promotion_declined_cycles(), + declined_before, + "an inadmissible full must not even take the measurement" + ); + assert!(crate::arena::pointer_in_nursery(survivor)); + // And the sweep did not invalidate dead young headers it was not asked + // to: that write is part of the promotion plan, not of every full. + assert_ne!( + unsafe { (*(user_to_header(dead) as *const GcHeader)).obj_type }, + 0, + "a full with no promotion plan must leave dead young headers as they were" + ); + }) + .join() + .expect("full-promotion test thread must not panic"); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 85f22246c4..a101f6c93b 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -26,6 +26,7 @@ mod forwarding_target_validation; mod forwarding_verification; mod fromspace_protect; mod fromspace_scan; +mod full_promotion; mod global_bootstrap; mod global_sink_isolation; mod handle_bound_method_name; @@ -47,6 +48,7 @@ mod lazy_tape_side_alloc; mod oldgen; mod os_tag; mod promote_in_place; +mod promoted_cohort; mod proxy_registry; mod retention_9628_9629; mod root_words; diff --git a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs new file mode 100644 index 0000000000..ff68f2e2b6 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs @@ -0,0 +1,56 @@ +//! #10182: old-reclaim is due once the promoted-but-unverified cohort exceeds +//! `max(floor, 2 × old live at last full)`, independently of the growth band +//! (which promotion credits blind) and of the retaining exemption. + +use super::super::policy::{ + old_reclaim_pressure_due, promoted_cohort_bound_bytes, seed_promoted_cohort_for_tests, + GC_MAJOR_PACING_RETAINING, +}; +use super::super::*; +use super::support::*; + +#[test] +fn promoted_cohort_bound_is_the_floor_or_twice_the_verified_live_set() { + let floor = gc_promoted_cohort_floor_dyn_bytes(); + assert_eq!(promoted_cohort_bound_bytes(0), floor); + assert_eq!(promoted_cohort_bound_bytes(floor / 4), floor); + assert_eq!(promoted_cohort_bound_bytes(floor), 2 * floor); + assert_eq!(promoted_cohort_bound_bytes(3 * floor), 6 * floor); +} + +#[test] +fn promoted_cohort_makes_old_reclaim_due_even_when_growth_reads_zero_and_the_heap_retains() { + let _isolation = GcTestIsolationGuard::new(); + let _pacing = crate::gc::policy::force_moving_gc_pacing(); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let previous_retaining = GC_MAJOR_PACING_RETAINING.with(std::cell::Cell::get); + let floor = gc_promoted_cohort_floor_dyn_bytes(); + // Growth reads zero (old_in_use == baseline) and the heap is classified + // retaining, so neither existing arm can fire: only the cohort can. + GC_MAJOR_PACING_RETAINING.with(|c| c.set(true)); + let occupancy = 10 * floor; + + seed_promoted_cohort_for_tests(floor - 1, 0); + assert!( + !old_reclaim_pressure_due(occupancy, occupancy), + "below the floor: not due" + ); + seed_promoted_cohort_for_tests(floor, 0); + assert!( + old_reclaim_pressure_due(occupancy, occupancy), + "at the floor: due" + ); + seed_promoted_cohort_for_tests(floor, floor); + assert!( + !old_reclaim_pressure_due(occupancy, occupancy), + "below 2 × live: not due" + ); + seed_promoted_cohort_for_tests(2 * floor, floor); + assert!( + old_reclaim_pressure_due(occupancy, occupancy), + "at 2 × live: due" + ); + + seed_promoted_cohort_for_tests(0, 0); + GC_MAJOR_PACING_RETAINING.with(|c| c.set(previous_retaining)); +} diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index c7498b1d77..d97364b8cd 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -307,7 +307,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after #10182 touched `gc/policy.rs`: two `Cell` byte counters (`GC_PROMOTED_SINCE_FULL`, `GC_OLD_LIVE_AT_LAST_FULL`), a pure bound function, one extra disjunct in `old_reclaim_pressure_due` (a trigger predicate evaluated before a cycle starts), a credit in `credit_promoted_bytes_to_old_baseline` (called after a minor completes) and a reset in `finish_full_old_reclaim_baseline` (called after the full's sweep). None of them runs inside the mark-complete → sweep-entry window of a synchronous full, and none allocates, relocates, or calls into JS.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -325,7 +325,7 @@ "crates/perry-runtime/src/gc/census.rs": "3f9e6be47b4454022b70ff2357bbdf3a80ef6a84c4986e58763acbdcce9142c1", "crates/perry-runtime/src/gc/cycle.rs": "77eadaf7c4157308c3b14be5e1ff11d7198b84b503a50ff2d244851248d3e800", "crates/perry-runtime/src/gc/mod.rs": "496d42cf8f41e4c71da6d50470ec1e796b065a919549b8239f579a5bc1a5b175", - "crates/perry-runtime/src/gc/policy.rs": "a701257f2e2310adabe16e33c0afcd935c7cd28e1ddc157b4974b48e2688cc4f", + "crates/perry-runtime/src/gc/policy.rs": "0ff322024f19dc0bac7e3e411f658a88874acf04f0a0c7e0af430da529a82345", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } } @@ -402,6 +402,18 @@ "verdict": "not_a_gc_pointer", "why": "Tiny-JSON completion countdown in a Cell. It stores only the number of bounded parse completions before the next arena-pressure poll (0..63), never an address or a NaN-boxed value." }, + { + "file": "crates/perry-runtime/src/gc/policy.rs", + "name": "GC_PROMOTED_SINCE_FULL", + "verdict": "not_a_gc_pointer", + "why": "#10182: a byte count — bytes promoted into old-gen since the last full old reclaim, read by the old-reclaim cohort bound. Holds no address." + }, + { + "file": "crates/perry-runtime/src/gc/policy.rs", + "name": "GC_OLD_LIVE_AT_LAST_FULL", + "verdict": "not_a_gc_pointer", + "why": "#10182: a byte count — the old-gen occupancy the last full old reclaim left behind, the verified live set the cohort bound is proportional to. Holds no address." + }, { "file": "crates/perry-runtime/src/gc/policy.rs", "name": "GC_TINY_PARSE_PRESSURE_BASE_BYTES", From 12a1fc70f160b3ec64ea761be86927e9716b8fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 15:20:02 +0200 Subject: [PATCH 2/5] gc: pre-empt the nursery minor with a promoting full; measure promotion against carried bytes; do not feed the full's census to the minor predictor (#10182) --- crates/perry-runtime/src/gc/cycle.rs | 85 +++++------ crates/perry-runtime/src/gc/mod.rs | 4 +- crates/perry-runtime/src/gc/oldgen.rs | 38 +---- .../src/gc/oldgen/sweep_cleanup.rs | 49 +++++++ crates/perry-runtime/src/gc/policy.rs | 40 +++++- .../perry-runtime/src/gc/promote_in_place.rs | 132 +++++++++++++----- .../src/gc/tests/full_promotion.rs | 18 ++- .../src/gc/tests/promoted_cohort.rs | 30 ++++ scripts/gc_runtime_root_holders.json | 20 +++ 9 files changed, 290 insertions(+), 126 deletions(-) diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index a00d50dc23..0ade2d14d2 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -655,40 +655,22 @@ impl GcCycleState { let progress_kind = trigger_kind.progress_kind(GcCollectionKind::Full); // #10182: a full mark-sweep is the one collection that can promote its // Eden survivors for free — it has already proven them live, and its - // own sweep has already reclaimed everything around them. The guards - // are the full-cycle-specific half of the decision; the shared policy - // guards live in `full_promotion_admissible`, and the measurement that - // decides it is taken after the sweep. - // - // * **Synchronous only.** A budgeted full opens mutator windows - // between steps. The retag hands every in-use young block to - // old-gen in one operation and `reset_young_after_promotion` then - // re-seats the inline bump allocator; an allocation landing between - // the two would be an allocation into a block that is registered - // old and owned by a young arena. - // * **Generational only.** Without the generational collector there - // is no young generation to promote and no barrier to uphold. - // * **Precise roots only.** A conservative stack scan's mark set is - // not a sound liveness measurement — the same exclusion - // `seed_promote_lock_from_sweep` applies two phases later, for the - // same reason. - // * **Old→young tracking complete.** Promotion makes these objects - // old; a later minor will only find their young children through - // the remembered set. This is the predicate the copying minor's - // `BarriersInactive` fallback tests, and it means the same here. - let promote_young_in_place = !progress_kind.is_budgeted() - && super::gen_gc_enabled() - && super::full_promotion_admissible() - && matches!( - super::roots::conservative_stack_scan_decision(), - super::roots::ConservativeStackScanDecision::SkipDisabled - ) - && super::barrier::old_to_young_tracking_complete(); + // own sweep has already reclaimed everything around them. Whether it + // MAY is `full_promotion_planned` (one gate, shared with the safepoint + // pre-emption); whether it DOES is decided after the sweep, from the + // measurement. + let promote_young_in_place = super::full_promotion_planned(progress_kind); let young_in_use_at_start = if promote_young_in_place { crate::arena::copying_from_space_in_use_bytes() } else { 0 }; + if promote_young_in_place && crate::gc::gc_diag_enabled() { + eprintln!( + "[gc-full-promote] plan young_bytes={young_in_use_at_start} survivor_bytes={}", + crate::arena::copying_active_survivor_in_use_bytes() + ); + } Self { collection_kind: GcCollectionKind::Full, trigger_kind, @@ -1660,17 +1642,38 @@ impl GcCycleState { } let young_bytes = self.young_in_use_at_start; let live_bytes = sweep.arena_live_from_space_bytes as usize; - // The most accurate young-survival figure the collector produces: a - // full trace's own census, not a prediction. Feed it to the copying - // minor's predictor whichever way the decision goes. - super::note_full_young_survival(young_bytes, live_bytes); - if !super::full_promotion_survival_holds_up(young_bytes, live_bytes) { + // The denominator is what the promotion would CARRY, not what the cycle + // started with: the sweep has already reset and released every young + // block that held nothing live, so those bytes cannot become old-gen + // garbage. What remains in use is exactly the set `retag_young_for_in_place_promotion` + // captures, and the threshold keeps the meaning it has on the copying + // minor — at most 5% of the promoted bytes are dead. Measured on + // `records_array_20m:parse`: 58.0 MB young at cycle start, 29.0 MB live, + // 29.7 MB still in use after the sweep (976‰ of what would be promoted). + // + // ★ The measurement is deliberately NOT fed to the copying minor's + // predictor (`note_young_survival`). The young census of a full is taken + // at whatever point the full runs, and on the #10182 rows that is a + // point where Eden still holds a parse result the JSON construction + // grace (`gc/json_defer.rs`) let die: 7‰ on `records_array_20m:roundtrip`, + // 333‰ on `records_array_8m:scan`. Fed to the predictor it turns the + // next minors into full evacuations of the following tree. Measured + // interleaved on the same binary (best of 3): 20m roundtrip 169.8 ms + // fed vs 131.0 ms not fed, 8m scan 415.2 ms / 168 MiB fed vs 338.0 ms / + // 155 MiB not fed, every other row of the 22-row matrix within noise. + let promotable_bytes = crate::arena::copying_from_space_in_use_bytes(); + if !super::full_promotion_survival_holds_up(promotable_bytes, live_bytes) { super::note_full_promotion_declined(); if crate::gc::gc_diag_enabled() { eprintln!( "[gc-full-promote] declined young_bytes={young_bytes} live_bytes={live_bytes} \ - survival_permille={}", - super::full_young_survival_permille(young_bytes, live_bytes), + promotable_bytes={promotable_bytes} carried_live_permille={} eden_live_bytes={} \ + eden_dead_bytes={} arena_live_bytes={} freed_bytes={}", + super::full_young_survival_permille(promotable_bytes, live_bytes), + sweep.eden_live_bytes, + sweep.eden_dead_bytes, + sweep.arena_live_bytes, + sweep.freed_bytes, ); } return; @@ -1701,12 +1704,14 @@ impl GcCycleState { } if crate::gc::gc_diag_enabled() { eprintln!( - "[gc-full-promote] promoted blocks={blocks} objects={} bytes={} \ - reserved_bytes={reserved_bytes} young_bytes={young_bytes} \ - live_bytes={live_bytes} survival_permille={} cycles={} declined={}", + "[gc-full-promote] promoted blocks={blocks} objects={} bytes={} live_objects={} \ + promoted_live_bytes={} reserved_bytes={reserved_bytes} young_bytes={young_bytes} \ + live_bytes={live_bytes} carried_live_permille={} cycles={} declined={}", stats.objects, stats.bytes, - super::full_young_survival_permille(young_bytes, live_bytes), + stats.live_objects, + stats.live_bytes, + super::full_young_survival_permille(promotable_bytes, live_bytes), super::full_promotion_cycles(), super::full_promotion_declined_cycles(), ); diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 62484e5505..b812980ba9 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -234,8 +234,8 @@ use promote_in_place::*; pub use promote_in_place::{ first_cycle_promotion_attempts, first_cycle_promotion_rollbacks, full_promoted_bytes, full_promoted_objects, full_promotion_cycles, full_promotion_declined_cycles, - in_place_promoted_objects, in_place_promotion_cycles, untraced_promoted_objects, - untraced_promotion_cycles, + in_place_promoted_objects, in_place_promotion_cycles, nursery_minors_preempted_by_full, + untraced_promoted_objects, untraced_promotion_cycles, }; /// Instrument-liveness counters (#7604): copying minors completed, objects /// relocated, loop back-edge polls reached. Mode-independent — they count what diff --git a/crates/perry-runtime/src/gc/oldgen.rs b/crates/perry-runtime/src/gc/oldgen.rs index 1f6d7b026c..2cc98c5bf5 100644 --- a/crates/perry-runtime/src/gc/oldgen.rs +++ b/crates/perry-runtime/src/gc/oldgen.rs @@ -1240,25 +1240,6 @@ impl IncrementalSweepState { self } - /// #10182: a synchronous full that intends to promote its young generation - /// in place must leave the young blocks in a state the promotion walk can - /// read. `finish_in_place_promotion`'s only liveness sources are the mark - /// bits (which this sweep clears as it goes) and "is this header's - /// `obj_type` arena-walkable" — so the sweep invalidates the headers of the - /// dead young objects it reclaims, exactly the way it already invalidates - /// dead OLD ones (`invalidate_dead_old_arena_header`). After that the - /// promotion's `PromotionLiveness::AssumeAllLive` walk registers precisely - /// the survivors, and the page-run description re-parses to the same count - /// because producer and expander apply the same `gc_type_is_arena_walkable` - /// filter. - /// - /// No `unregister_old_object_pages` here: these objects were young, so they - /// were never in the old-gen page index to be removed from. - pub(super) fn invalidating_dead_young_headers(mut self, on: bool) -> Self { - self.arena.invalidate_dead_young_headers = on; - self - } - pub(super) fn step(&mut self, budget: usize) -> bool { match self.subphase { SweepCycleSubphase::CollectionSideBuffers => { @@ -1411,8 +1392,7 @@ struct ArenaSweepObjectsState { /// #7901: see `SweepTraceStats::arena_live_from_space_bytes`. arena_live_from_space_bytes: u64, active_survivor_blocks: std::ops::Range, - /// #10182: see `IncrementalSweepState::invalidating_dead_young_headers`. - invalidate_dead_young_headers: bool, + invalidate_dead_young_headers: bool, // #10182: see `sweep_cleanup.rs` } impl ArenaSweepObjectsState { @@ -1673,20 +1653,8 @@ impl ArenaSweepObjectsState { finalize_dead_arena_payload(header, user_ptr, self.overflow_active); if self.reclaim_dead_old_blocks && dead_old { self.pending_old_unregister.defer(header, total_size); - } else if self.invalidate_dead_young_headers - && crate::arena::block_in_copying_from_space( - block_idx, - self.resettable_general_n, - &self.active_survivor_blocks, - ) - { - // #10182: this block is about to be handed to old-gen whole. Leave - // the header unparseable as an object so the promotion walk and the - // page-run expansion both skip it. `size` is deliberately preserved - // — every arena walker hops by it, this one included. - (*header).obj_type = 0; - (*header).gc_flags = 0; - (*header)._reserved = 0; + } else { + self.invalidate_dead_young_header_for_promotion(header, block_idx); } } } diff --git a/crates/perry-runtime/src/gc/oldgen/sweep_cleanup.rs b/crates/perry-runtime/src/gc/oldgen/sweep_cleanup.rs index b14e62fd85..47b5b3aa3a 100644 --- a/crates/perry-runtime/src/gc/oldgen/sweep_cleanup.rs +++ b/crates/perry-runtime/src/gc/oldgen/sweep_cleanup.rs @@ -81,3 +81,52 @@ impl ArenaSweepCleanupState { self.stats } } + +impl IncrementalSweepState { + /// #10182: a synchronous full that intends to promote its young generation + /// in place must leave the young blocks in a state the promotion walk can + /// read. `finish_in_place_promotion`'s only liveness sources are the mark + /// bits (which this sweep clears as it goes) and "is this header's + /// `obj_type` arena-walkable" — so the sweep invalidates the headers of the + /// dead young objects it reclaims, exactly the way it already invalidates + /// dead OLD ones (`invalidate_dead_old_arena_header`). After that the + /// promotion's `PromotionLiveness::AssumeAllLive` walk registers precisely + /// the survivors, and the page-run description re-parses to the same count + /// because producer and expander apply the same `gc_type_is_arena_walkable` + /// filter. + /// + /// Here rather than in `oldgen.rs` for the 2000-line file cap. + pub(in crate::gc) fn invalidating_dead_young_headers(mut self, on: bool) -> Self { + self.arena.invalidate_dead_young_headers = on; + self + } +} + +impl ArenaSweepObjectsState { + /// #10182: the per-object half of [`IncrementalSweepState::invalidating_dead_young_headers`]. + /// + /// Only young (from-space: Eden + active survivor) blocks, only when the + /// cycle planned a promotion. No `unregister_old_object_pages`: these + /// objects were young, so they were never in the old-gen page index. `size` + /// is deliberately preserved — every arena walker hops by it, the promotion + /// walk and the page-run expansion included. + #[inline] + pub(super) unsafe fn invalidate_dead_young_header_for_promotion( + &self, + header: *mut GcHeader, + block_idx: usize, + ) { + if !self.invalidate_dead_young_headers + || !crate::arena::block_in_copying_from_space( + block_idx, + self.resettable_general_n, + &self.active_survivor_blocks, + ) + { + return; + } + (*header).obj_type = 0; + (*header).gc_flags = 0; + (*header)._reserved = 0; + } +} diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 99e14eabec..3cad647c32 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1973,7 +1973,16 @@ pub(super) fn promoted_cohort_bound_bytes(old_live_at_last_full: usize) -> usize #[inline] fn promoted_cohort_bound_due() -> bool { - GC_PROMOTED_SINCE_FULL.with(Cell::get) + promoted_cohort_bound_due_with(0) +} + +/// [`promoted_cohort_bound_due`] as it will read once `pending_promotion_bytes` +/// more have been promoted — see `promoting_full_preempts_nursery_minor`. +#[inline] +pub(super) fn promoted_cohort_bound_due_with(pending_promotion_bytes: usize) -> bool { + GC_PROMOTED_SINCE_FULL + .with(Cell::get) + .saturating_add(pending_promotion_bytes) >= promoted_cohort_bound_bytes(GC_OLD_LIVE_AT_LAST_FULL.with(Cell::get)) } @@ -3405,6 +3414,35 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { // mirror would leave the back-edge poll armed forever. set_safepoint_pending(false); let _declared = DeclaredSafepointGuard::enter(); + // #10182: a nursery collection whose promotion would make the promoted- + // cohort bound due runs as the full that bound would schedule next, while + // the young generation is still here for it to promote in place. See + // `promoting_full_preempts_nursery_minor`. + if !scheduled + && matches!( + due, + Some(BudgetedGcTrigger::ArenaBytes | BudgetedGcTrigger::YoungScavengeCap) + ) + && !GC_OLD_RECLAIM_IN_PROGRESS.with(Cell::get) + && super::promoting_full_preempts_nursery_minor() + { + let _reentry = OldReclaimReentryGuard::enter(); + GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(false)); + super::note_nursery_minor_preempted_by_full(); + super::diag_sites::trigger_decision("safepoint", "PromotingFull"); + super::diag_sites::set_full_site("safepoint_promoting_full"); + // No `force_full_scan`: roots are precise at this safepoint, which is + // one of the conditions the pre-emption checked. + let pre_in_use = crate::arena::arena_in_use_bytes(); + let outcome = gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( + GcTriggerKind::OldGenBytes, + )); + // It stood in for the nursery collection, so it re-baselines the + // nursery trigger the way that collection would have. + gc_finish_arena_trigger_collection(pre_in_use, outcome); + super::record_safepoint_drain(super::SafepointDrainKind::OldReclaim); + return true; + } let kind = match due { // #7909: the nursery cap and the whole-arena trigger are the same // collection here — this IS the evacuating collector the cap is for. diff --git a/crates/perry-runtime/src/gc/promote_in_place.rs b/crates/perry-runtime/src/gc/promote_in_place.rs index 7558ae595f..9918b5dc79 100644 --- a/crates/perry-runtime/src/gc/promote_in_place.rs +++ b/crates/perry-runtime/src/gc/promote_in_place.rs @@ -201,6 +201,7 @@ thread_local! { static FULL_PROMOTION_DECLINED_CYCLES: Cell = const { Cell::new(0) }; static FULL_PROMOTED_OBJECTS: Cell = const { Cell::new(0) }; static FULL_PROMOTED_BYTES: Cell = const { Cell::new(0) }; + static NURSERY_MINORS_PREEMPTED_BY_FULL: Cell = const { Cell::new(0) }; } /// Record that `bytes` of young capacity became old-gen, so the next @@ -401,26 +402,95 @@ pub(super) fn full_promotion_admissible() -> bool { in_place_promotion_admissible() } -/// Did the full's own sweep measure the young generation as live enough to -/// promote it whole? (#10182) +/// The whole "may this full plan a promotion" gate, shared by the cycle +/// constructor and by [`promoting_full_preempts_nursery_minor`] so the two can +/// never disagree — a pre-emption the full then refuses to act on would leave +/// Eden full and the predicate still true at the next safepoint. +/// +/// * **Synchronous only.** A budgeted full opens mutator windows between +/// steps; the retag hands every in-use young block to old-gen in one +/// operation and `reset_young_after_promotion` then re-seats the inline +/// bump allocator, and an allocation between the two would land in a block +/// registered old and owned by a young arena. +/// * **Generational only.** Without it there is no young generation to +/// promote and no barrier to uphold. +/// * **Precise roots only.** A conservative scan's mark set is not a sound +/// liveness measurement — the exclusion `seed_promote_lock_from_sweep` +/// already applies to the same sweep census, for the same reason. +/// * **Old→young tracking complete.** Promotion makes these objects old, and +/// a later minor finds their young children only through the remembered +/// set: the predicate behind the copying minor's `BarriersInactive` +/// fallback. +pub(super) fn full_promotion_planned(progress_kind: super::GcProgressKind) -> bool { + !progress_kind.is_budgeted() + && super::gen_gc_enabled() + && full_promotion_admissible() + && matches!( + super::roots::conservative_stack_scan_decision(), + super::roots::ConservativeStackScanDecision::SkipDisabled + ) + && super::barrier::old_to_young_tracking_complete() +} + +/// Should the nursery collection due at this precise safepoint be a full that +/// promotes in place, rather than a copying minor followed by a full? (#10182) +/// +/// The promoted-cohort bound (`policy::promoted_cohort_bound_bytes`) can only +/// become due AFTER a promotion lands, i.e. after the copying minor has handed +/// the young generation to old-gen. Its full then runs at the next allocation +/// point — behind a conservative scan, over an EMPTY young generation, with +/// nothing left to promote. Measured on `records_array_8m:scan` with the bound +/// alone: 4 minors, 2 fulls, both fulls at `site=alloc_point` with +/// `from_space` ≤ 1376 bytes, and the in-place promotion at the full never ran +/// once. Two collections doing the work of one, and the second one blind. +/// +/// So ask the question one collection earlier, while the tree is still in +/// Eden: would the promotion this minor is about to perform make the cohort +/// bound due? If so, run the full now. It traces the young generation anyway, +/// so it promotes it itself — verified rather than assumed — and it sweeps +/// the old cohort the bound exists to reclaim, in the same pass. +/// +/// Conditions, each load-bearing: +/// * the copying minor's own predictor says it would promote the whole young +/// generation (`should_promote_young_in_place`) — otherwise the minor +/// evacuates, the cohort does not grow, and there is nothing to pre-empt; +/// * the full would be allowed to plan the promotion +/// ([`full_promotion_planned`], the same gate the cycle reads) — otherwise +/// this is a full that leaves Eden as it found it; +/// * `promoted_since_full + young_in_use` reaches the bound — the same +/// predicate the bound itself evaluates, one promotion ahead. +/// +/// Not a #7594-style prediction of pressure the full cannot relieve: the +/// promotable bytes it counts are the ones this full promotes, and the old-gen +/// cohort it reclaims is already there. +pub(super) fn promoting_full_preempts_nursery_minor() -> bool { + if !should_promote_young_in_place() + || !full_promotion_planned(super::GcProgressKind::LegacySynchronous) + { + return false; + } + super::policy::promoted_cohort_bound_due_with(crate::arena::copying_from_space_in_use_bytes()) +} + +/// Did the full's own sweep leave young blocks live enough to promote whole? +/// (#10182) /// -/// `young_bytes` is from-space occupancy captured at cycle start, -/// `live_bytes` the from-space share of the sweep's live census. The threshold -/// is [`PROMOTE_SURVIVAL_THRESHOLD_PERMILLE`], the same 95% the copying minor -/// uses, and it means the same thing: at most 5% of the promoted bytes are -/// dead, and they cost footprint until the next full. +/// `carried_bytes` is young (from-space) occupancy AFTER the sweep — the bytes +/// the promotion would hand to old-gen, since the sweep has already reset every +/// young block with nothing live on it — and `live_bytes` the from-space share +/// of the sweep's live census. The threshold is +/// [`PROMOTE_SURVIVAL_THRESHOLD_PERMILLE`], the same 95% the copying minor +/// uses, and it bounds the same thing: at most 5% of the promoted bytes are +/// dead and cost footprint until the next full. /// /// ★ The number this tests is a MEASUREMENT, not a prediction. The copying /// minor reads the *previous* cycle's ratio because it must decide before it -/// traces; this reads the ratio this very cycle just measured. A misprediction -/// is therefore impossible here — what remains is the ordinary policy question -/// of whether a 95%-live nursery is worth tenuring whole, and the answer is -/// the one #7742 already argued. -pub(super) fn full_promotion_survival_holds_up(young_bytes: usize, live_bytes: usize) -> bool { - if young_bytes == 0 { +/// traces; this reads what this very cycle just traced and swept. +pub(super) fn full_promotion_survival_holds_up(carried_bytes: usize, live_bytes: usize) -> bool { + if carried_bytes == 0 { return false; } - full_young_survival_permille(young_bytes, live_bytes) >= PROMOTE_SURVIVAL_THRESHOLD_PERMILLE + full_young_survival_permille(carried_bytes, live_bytes) >= PROMOTE_SURVIVAL_THRESHOLD_PERMILLE } /// The ratio itself, in permille, clamped to 1000. Shared by the decision and @@ -433,30 +503,6 @@ pub(super) fn full_young_survival_permille(young_bytes: usize, live_bytes: usize .min(1000) } -/// Record a full's exact young-survival measurement for the copying minor's -/// predictor (#10182). -/// -/// Deliberately NOT [`note_young_survival`]. That function does three other -/// things — it settles the untraced run, re-bases the untraced budget against -/// the current old-gen, and can raise an old-reclaim request — all of which a -/// full either does itself or makes meaningless: `finish_full_old_reclaim_baseline` -/// runs `note_full_collection_reclaimed_old_gen` moments later and resets both -/// the untraced run and the budget base, and clears any old-reclaim request. -/// Calling it would therefore be a no-op wrapped around a redundant request, -/// and the redundant request is the kind that schedules a second full behind -/// the one that just ran. -/// -/// What IS worth keeping is the ratio: it is the most accurate young-survival -/// figure the collector ever produces, and the next copying minor's in-place -/// decision reads exactly this cell. -pub(super) fn note_full_young_survival(young_bytes: usize, live_bytes: usize) { - if young_bytes == 0 { - return; - } - LAST_YOUNG_SURVIVAL_PERMILLE - .with(|c| c.set(Some(full_young_survival_permille(young_bytes, live_bytes)))); -} - /// A full mark-sweep promoted its Eden survivors in place (#10182). pub(super) fn note_full_promotion(promoted_bytes: usize, promoted_objects: usize) { FULL_PROMOTION_CYCLES.with(|c| c.set(c.get().saturating_add(1))); @@ -491,6 +537,16 @@ pub fn full_promoted_bytes() -> usize { FULL_PROMOTED_BYTES.with(Cell::get) } +pub(super) fn note_nursery_minor_preempted_by_full() { + NURSERY_MINORS_PREEMPTED_BY_FULL.with(|c| c.set(c.get().saturating_add(1))); +} + +/// How many nursery-due safepoints ran a promoting full instead of a copying +/// minor ([`promoting_full_preempts_nursery_minor`]). +pub fn nursery_minors_preempted_by_full() -> u64 { + NURSERY_MINORS_PREEMPTED_BY_FULL.with(Cell::get) +} + /// The untraced-promotion budget, and therefore **the worst-case retained /// garbage of the untraced path**: every byte it admits is *assumed* live, so /// after an abrupt live→dead phase change every one of them can be garbage diff --git a/crates/perry-runtime/src/gc/tests/full_promotion.rs b/crates/perry-runtime/src/gc/tests/full_promotion.rs index 7226b0f6be..2afc1e5006 100644 --- a/crates/perry-runtime/src/gc/tests/full_promotion.rs +++ b/crates/perry-runtime/src/gc/tests/full_promotion.rs @@ -16,7 +16,8 @@ //! walking the index, because the `debug_assert` inside the run expansion is //! compiled out of the `--release` test build this suite runs under. //! 4. **Both states of the decision**: a mostly-dead nursery declines (and says -//! so in its own counter), and a thread that is not admitted never plans. +//! so in its own counter, without overwriting the copying minor's +//! predictor), and a thread that is not admitted never plans. //! //! Each test runs on a fresh thread: the arenas are thread-local, so that is //! what makes "survival" a property of the heap the test built rather than of @@ -106,11 +107,6 @@ fn a_full_over_a_live_nursery_promotes_it_in_place_and_leaves_no_young_generatio promoted > LIVE_LEAVES as u64, "the promotion must have covered the array and all {LIVE_LEAVES} leaves, got {promoted}" ); - assert!( - super::super::promote_in_place::last_young_survival_permille() - .is_some_and(|p| p >= 950), - "the full's own measurement must have been fed to the predictor" - ); // (1) No young generation remains. assert_eq!( @@ -197,6 +193,7 @@ fn a_full_over_a_mostly_dead_nursery_declines_and_keeps_its_young_generation() { let cycles_before = crate::gc::full_promotion_cycles(); let declined_before = crate::gc::full_promotion_declined_cycles(); + let predictor_before = super::super::promote_in_place::last_young_survival_permille(); full_collect(); assert_eq!( @@ -205,10 +202,11 @@ fn a_full_over_a_mostly_dead_nursery_declines_and_keeps_its_young_generation() { "the full was admitted, measured a mostly-dead nursery, and must say it declined" ); assert_eq!(crate::gc::full_promotion_cycles(), cycles_before); - assert!( - super::super::promote_in_place::last_young_survival_permille() - .is_some_and(|p| p < 950), - "the declining measurement must still reach the copying minor's predictor" + assert_eq!( + super::super::promote_in_place::last_young_survival_permille(), + predictor_before, + "a full's young census is taken wherever the full lands and is NOT the \ + copying minor's measurement to overwrite (see `maybe_promote_young_after_full`)" ); assert!( crate::arena::pointer_in_nursery(survivor), diff --git a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs index ff68f2e2b6..f9fe8073f8 100644 --- a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs @@ -54,3 +54,33 @@ fn promoted_cohort_makes_old_reclaim_due_even_when_growth_reads_zero_and_the_hea seed_promoted_cohort_for_tests(0, 0); GC_MAJOR_PACING_RETAINING.with(|c| c.set(previous_retaining)); } + +#[test] +fn the_cohort_floor_is_one_base_nursery_cap() { + // Unconstrained process: the budget-scaled accessor returns its default. + if gc_heap_budget_bytes().is_none() { + assert_eq!( + gc_promoted_cohort_floor_dyn_bytes(), + super::super::policy::gc_scavenge_nursery_cap_bytes(), + "the floor is one nursery quantum, not a separate constant" + ); + } +} + +#[test] +fn the_cohort_bound_can_be_asked_one_promotion_ahead() { + // `promoting_full_preempts_nursery_minor` asks whether the promotion a + // minor is ABOUT to perform would make the bound due. The pending bytes + // must count exactly like already-promoted bytes, at both edges. + let _isolation = GcTestIsolationGuard::new(); + let floor = gc_promoted_cohort_floor_dyn_bytes(); + seed_promoted_cohort_for_tests(floor / 2, 0); + assert!(!super::super::policy::promoted_cohort_bound_due_with(0)); + assert!(!super::super::policy::promoted_cohort_bound_due_with( + floor - floor / 2 - 1 + )); + assert!(super::super::policy::promoted_cohort_bound_due_with( + floor - floor / 2 + )); + seed_promoted_cohort_for_tests(0, 0); +} diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index d97364b8cd..3856a78d6e 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -3420,6 +3420,26 @@ "file": "crates/perry-runtime/src/gc/promote_in_place.rs", "name": "IN_PLACE_PROMOTED_OBJECTS" }, + { + "file": "crates/perry-runtime/src/gc/promote_in_place.rs", + "name": "FULL_PROMOTED_BYTES" + }, + { + "file": "crates/perry-runtime/src/gc/promote_in_place.rs", + "name": "FULL_PROMOTED_OBJECTS" + }, + { + "file": "crates/perry-runtime/src/gc/promote_in_place.rs", + "name": "FULL_PROMOTION_CYCLES" + }, + { + "file": "crates/perry-runtime/src/gc/promote_in_place.rs", + "name": "FULL_PROMOTION_DECLINED_CYCLES" + }, + { + "file": "crates/perry-runtime/src/gc/promote_in_place.rs", + "name": "NURSERY_MINORS_PREEMPTED_BY_FULL" + }, { "file": "crates/perry-runtime/src/gc/promote_in_place.rs", "name": "IN_PLACE_PROMOTION_CYCLES" From 46679647b7353052c71b1d74e05b2eb023683a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 15:46:23 +0200 Subject: [PATCH 3/5] gc: consult the promoted-cohort bound only at the nursery safepoint, never as an old-reclaim arm (#10182) --- crates/perry-runtime/src/gc/policy.rs | 24 +++++++----- .../src/gc/tests/promoted_cohort.rs | 38 +++++++------------ 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 3cad647c32..991b0485a7 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1745,9 +1745,17 @@ pub(super) fn old_reclaim_pressure_due(old_in_use: usize, baseline: usize) -> bo let crossed_absolute_threshold = old_in_use >= threshold && baseline < threshold && !GC_MAJOR_PACING_RETAINING.with(|c| c.get()); + // #10182: the promoted-cohort bound is deliberately NOT a disjunct here. + // As one it schedules exactly the full #7592 and #7965 pin as futile — a + // full that becomes due only because a promotion just landed, reached at + // the next allocation point behind a conservative scan with the young + // generation already empty. Measured with it: `12_large_live_set` gained + // one such full (+55% wall) and `records_array_20m:parse` paid 115 ms for + // one. The bound is consulted one promotion EARLIER instead, at the + // nursery safepoint, where the full it justifies can promote the young + // generation itself (`promoting_full_preempts_nursery_minor`). crossed_absolute_threshold || old_in_use.saturating_sub(baseline) >= gc_old_reclaim_growth_band_bytes(baseline) - || promoted_cohort_bound_due() } /// Whether an imminent promotion justifies a full old reclaim FIRST. @@ -1938,8 +1946,9 @@ pub(super) fn credit_promoted_bytes_to_old_baseline(promoted_bytes: usize) { GC_PROMOTED_SINCE_FULL.with(|bytes| bytes.set(bytes.get().saturating_add(promoted_bytes))); } -/// #10182: how many promoted-but-unverified bytes old-reclaim may leave -/// unexamined before a full is due: `max(floor, k × old live at last full)`. +/// #10182: how many promoted-but-unverified bytes may accumulate before the +/// nursery collection that would add more runs as a promoting full instead: +/// `max(floor, k × old live at last full)`. /// /// The growth band cannot see this cohort: promotion credits the old baseline /// (#7965, and rightly — a pinned baseline degenerates the band), so a loop @@ -1971,13 +1980,8 @@ pub(super) fn promoted_cohort_bound_bytes(old_live_at_last_full: usize) -> usize .max(old_live_at_last_full.saturating_mul(PROMOTED_COHORT_GROWTH_MULTIPLIER)) } -#[inline] -fn promoted_cohort_bound_due() -> bool { - promoted_cohort_bound_due_with(0) -} - -/// [`promoted_cohort_bound_due`] as it will read once `pending_promotion_bytes` -/// more have been promoted — see `promoting_full_preempts_nursery_minor`. +/// Is the promoted cohort at its bound once `pending_promotion_bytes` more have +/// been promoted? Read only by `promoting_full_preempts_nursery_minor`. #[inline] pub(super) fn promoted_cohort_bound_due_with(pending_promotion_bytes: usize) -> bool { GC_PROMOTED_SINCE_FULL diff --git a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs index f9fe8073f8..3c89c2c623 100644 --- a/crates/perry-runtime/src/gc/tests/promoted_cohort.rs +++ b/crates/perry-runtime/src/gc/tests/promoted_cohort.rs @@ -1,6 +1,7 @@ -//! #10182: old-reclaim is due once the promoted-but-unverified cohort exceeds -//! `max(floor, 2 × old live at last full)`, independently of the growth band -//! (which promotion credits blind) and of the retaining exemption. +//! #10182: the promoted-but-unverified cohort bound, `max(floor, 2 × old live +//! at last full)` with a floor of one base nursery cap. It is read one promotion +//! ahead by the nursery-safepoint pre-emption and is never an arm of +//! `old_reclaim_pressure_due` (which would reinstate #7592's futile full). use super::super::policy::{ old_reclaim_pressure_due, promoted_cohort_bound_bytes, seed_promoted_cohort_for_tests, @@ -19,38 +20,27 @@ fn promoted_cohort_bound_is_the_floor_or_twice_the_verified_live_set() { } #[test] -fn promoted_cohort_makes_old_reclaim_due_even_when_growth_reads_zero_and_the_heap_retains() { +fn the_cohort_bound_never_makes_old_reclaim_due_on_its_own() { + // #7592/#7965: a full that is due only because promotion moved bytes into + // old-gen frees nothing. The cohort bound must not reintroduce that arm — + // however large the unverified cohort, growth pacing reads zero here and + // only the safepoint pre-emption may act on the bound. let _isolation = GcTestIsolationGuard::new(); let _pacing = crate::gc::policy::force_moving_gc_pacing(); let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); let previous_retaining = GC_MAJOR_PACING_RETAINING.with(std::cell::Cell::get); - let floor = gc_promoted_cohort_floor_dyn_bytes(); - // Growth reads zero (old_in_use == baseline) and the heap is classified - // retaining, so neither existing arm can fire: only the cohort can. GC_MAJOR_PACING_RETAINING.with(|c| c.set(true)); + let floor = gc_promoted_cohort_floor_dyn_bytes(); let occupancy = 10 * floor; - - seed_promoted_cohort_for_tests(floor - 1, 0); + seed_promoted_cohort_for_tests(100 * floor, 0); assert!( - !old_reclaim_pressure_due(occupancy, occupancy), - "below the floor: not due" + super::super::policy::promoted_cohort_bound_due_with(0), + "precondition: the cohort is far past its bound" ); - seed_promoted_cohort_for_tests(floor, 0); - assert!( - old_reclaim_pressure_due(occupancy, occupancy), - "at the floor: due" - ); - seed_promoted_cohort_for_tests(floor, floor); assert!( !old_reclaim_pressure_due(occupancy, occupancy), - "below 2 × live: not due" - ); - seed_promoted_cohort_for_tests(2 * floor, floor); - assert!( - old_reclaim_pressure_due(occupancy, occupancy), - "at 2 × live: due" + "the cohort bound alone must not schedule an old reclaim" ); - seed_promoted_cohort_for_tests(0, 0); GC_MAJOR_PACING_RETAINING.with(|c| c.set(previous_retaining)); } From ba8c072744f35c0115466caf9febe6ed7acdfb9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 15:52:37 +0200 Subject: [PATCH 4/5] lint(gc): re-audit and re-pin the census window for #10182 --- scripts/gc_runtime_root_holders.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 3856a78d6e..c577f49142 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -307,7 +307,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after #10182 touched `gc/policy.rs`: two `Cell` byte counters (`GC_PROMOTED_SINCE_FULL`, `GC_OLD_LIVE_AT_LAST_FULL`), a pure bound function, one extra disjunct in `old_reclaim_pressure_due` (a trigger predicate evaluated before a cycle starts), a credit in `credit_promoted_bytes_to_old_baseline` (called after a minor completes) and a reset in `finish_full_old_reclaim_baseline` (called after the full's sweep). None of them runs inside the mark-complete → sweep-entry window of a synchronous full, and none allocates, relocates, or calls into JS.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after #10182 touched `gc/policy.rs`: two `Cell` byte counters (`GC_PROMOTED_SINCE_FULL`, `GC_OLD_LIVE_AT_LAST_FULL`), a pure bound function, one extra disjunct in `old_reclaim_pressure_due` (a trigger predicate evaluated before a cycle starts), a credit in `credit_promoted_bytes_to_old_baseline` (called after a minor completes) and a reset in `finish_full_old_reclaim_baseline` (called after the full's sweep). None of them runs inside the mark-complete → sweep-entry window of a synchronous full, and none allocates, relocates, or calls into JS. Re-audited 2026-09-13 for #10182's full-collection promotion, superseding the earlier same-day cohort-bound note. `gc/cycle.rs`: `new_full` evaluates `full_promotion_planned` and reads young occupancy BEFORE mark propagation starts; `step_sweep` chains `invalidating_dead_young_headers` onto the sweep state, which is constructed after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS; `maybe_promote_young_after_full` (retag + `finish_in_place_promotion`, which moves blocks between arenas but relocates no object) runs after the sweep completes; `publish_reclaim_outcome` only changes the from-space live figure it publishes. `gc/mod.rs` gains re-exports of counters. `gc/policy.rs` gains two byte counters, a pure bound function, a credit/reset of those counters outside any cycle, and a pre-emption at the nursery safepoint that decides WHICH collection to start, before it starts. None of this runs between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`, allocates GC objects there, relocates, or calls into JS.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -323,9 +323,9 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "3f9e6be47b4454022b70ff2357bbdf3a80ef6a84c4986e58763acbdcce9142c1", - "crates/perry-runtime/src/gc/cycle.rs": "77eadaf7c4157308c3b14be5e1ff11d7198b84b503a50ff2d244851248d3e800", - "crates/perry-runtime/src/gc/mod.rs": "496d42cf8f41e4c71da6d50470ec1e796b065a919549b8239f579a5bc1a5b175", - "crates/perry-runtime/src/gc/policy.rs": "0ff322024f19dc0bac7e3e411f658a88874acf04f0a0c7e0af430da529a82345", + "crates/perry-runtime/src/gc/cycle.rs": "6585dd7cffc76b660e5e1e232fce1511a02ab8dec0d17e967526750e4359f624", + "crates/perry-runtime/src/gc/mod.rs": "c5dc0268dc0b57124a84638cd5ac073eb10a6ac0ea066c89c7bc0201ce353fe6", + "crates/perry-runtime/src/gc/policy.rs": "f1c0af223eac9977bdb0ec8b57e01d2e8dc58ab351f6f19c8aa2e95a6a3f97dc", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } } From 6796cd3328e78253ee12a7de4acc5b59a4d96f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 15:53:14 +0200 Subject: [PATCH 5/5] changelog: fragment for #10204 --- changelog.d/10204-gc-full-promotes-eden-survivors.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog.d/10204-gc-full-promotes-eden-survivors.md diff --git a/changelog.d/10204-gc-full-promotes-eden-survivors.md b/changelog.d/10204-gc-full-promotes-eden-survivors.md new file mode 100644 index 0000000000..3a09f0895f --- /dev/null +++ b/changelog.d/10204-gc-full-promotes-eden-survivors.md @@ -0,0 +1,10 @@ +**gc: a synchronous full mark-sweep promotes its young generation in place, and a nursery collection whose promotion would exceed a promoted-cohort bound runs as that full (#10182). Draft: does NOT meet its acceptance bar — see below.** + +Mechanism: + +- `GcCycleState::maybe_promote_young_after_full` (`gc/cycle.rs`): after the full's sweep and before its remembered-set clear, a synchronous, generational, precise-root full with complete old→young tracking retags every in-use Eden/survivor block to old-gen and finishes the promotion through the copying minor's `arena/promote.rs` machinery (`retag_young_for_in_place_promotion` / `finish_in_place_promotion`, `PromotionLiveness::AssumeAllLive`). It promotes only when the sweep's live from-space bytes are ≥ 95% (`PROMOTE_SURVIVAL_THRESHOLD_PERMILLE`) of the young bytes still in use after the sweep — the bytes the promotion would carry. The sweep invalidates the headers of dead young objects when a promotion is planned (`IncrementalSweepState::invalidating_dead_young_headers`), so the described page runs index exactly the survivors. A promoting full publishes a zero from-space live share (#7901) and leaves no young generation. +- Promoted-cohort bound (`gc/policy.rs`): bytes promoted since the last full, bounded by `max(one base nursery cap, 2 × old live at the last full)`. It is NOT an arm of `old_reclaim_pressure_due` (that reinstates #7592/#7965's futile full and breaks the two tests pinning it); `promoting_full_preempts_nursery_minor` consults it one promotion ahead at the nursery safepoint. +- The full's young census is not fed to the copying minor's predictor: measured to turn later minors into evacuations (20m roundtrip 131 → 170 ms, 8m scan 338 → 415 ms) with no RSS gain. +- Counters: `full_promotion_cycles`, `full_promotion_declined_cycles`, `full_promoted_objects`, `full_promoted_bytes`, `nursery_minors_preempted_by_full`; diag lines `[gc-full-promote] plan|promoted|declined`, `[gc-trigger] kind=PromotingFull`, `promoted_since_full=` / `cohort_bound=`. + +Measured (JSON matrix, 22 rows, interleaved best of 3, same tree, loaded host): `records_array_20m:roundtrip` 189 ms / 284 MiB → 128 ms / 193 MiB (node/bun best 162 / 261); 8m parse/sparse RSS 109 → 97–98 MiB at equal CPU; but `records_array_20m` parse/scan/sparse and `records_object_20m:parse` regress 1.5–2.5× CPU (past the node/bun best) and `records_array_8m:scan` 1.8–2.1× CPU. The fulls that pay for the RSS mark a live 29–58 MB tree; that mark alone exceeds the rows' CPU lead. gc-ratchet: `14_grow_then_churn` changes seven gated counters (its 1 MB nursery env makes the cohort floor 1 MB, so pre-emption replaces 4 minors with declined fulls: peak RSS −59%, wall ×2.1); `11_collect_at_depth` copies 0.2% fewer objects (manual `gc()` fulls now promote); every other probe's gated counters unchanged.