perf(gc): birth a large JSON result in the arena when the young generation already holds a document

changelog.d/10177-json-leaf-arena-route.md

@@ -0,0 +1,3 @@
+### perf(gc): birth a large JSON result in the arena when the young generation already holds a document
+
+JSON results at or above 512 KB are malloc-tracked so the next minor can reclaim a discarded result cheaply. On a parse-then-stringify loop over a document-sized input that inverted: the non-empty malloc registry forbids the untraced in-place promotion, so every minor traced the whole freshly parsed tree (55 ms for 20 MB, half the roundtrip's wall time) to reclaim one leaf. Such a leaf is now born old in the arena when the young generation holds at least the leaf's bytes and a base nursery's worth of data and was not measured as dying, and the next trigger decision gives the nursery minor one-time priority over old-reclaim while the young generation is unmeasured, so a stringify-only loop still promotes its input once and then keeps the malloc path. Measured on `records_array_20m:roundtrip`: CPU 206.8 → 114.1 ms (0.55×, now ahead of the better of Node 26.5.1 / Bun 1.3.14 at 162.1) and peak RSS 295 → 266 MiB (Node 261); 20 MB stringify rows −40 MB peak RSS at flat CPU; every other JSON row within noise (#10169).

crates/perry-runtime/src/gc/mod.rs

@@ -39,9 +39,11 @@ mod json_defer;
 mod policy;
 pub(crate) use json_defer::JsonParseAllocation;
 pub(crate) use policy::gc_runtime_safepoint;
+pub(crate) use policy::note_young_leaf_born_old;
 /// The one writer of `GC_SAFEPOINT_PENDING` — it also keeps the poll's global
 /// arming shadow in step. See `gc/poll_arm.rs`.
 pub(crate) use policy::set_safepoint_pending;
+pub(crate) use policy::young_generation_holds_a_nursery;
 pub use policy::*;
 mod progress;
 pub use progress::*;
@@ -231,10 +233,17 @@ mod native_stack_scan;
 /// mechanism is `arena/promote.rs`; this decides when to use it.
 mod promote_in_place;
 use promote_in_place::*;
+#[cfg(test)]
+pub(crate) use promote_in_place::{
+    clear_young_survival_for_tests, last_young_survival_permille, seed_young_survival_for_tests,
+};
 pub use promote_in_place::{
     first_cycle_promotion_attempts, first_cycle_promotion_rollbacks, in_place_promoted_objects,
     in_place_promotion_cycles, untraced_promoted_objects, untraced_promotion_cycles,
 };
+pub(crate) use promote_in_place::{
+    young_generation_measured_dying, young_generation_measured_retained,
+};
 /// Instrument-liveness counters (#7604): copying minors completed, objects
 /// relocated, loop back-edge polls reached. Mode-independent — they count what
 /// the COLLECTOR did, not what forced it, so they outlive any one stress knob.

crates/perry-runtime/src/gc/policy.rs

@@ -120,6 +120,19 @@ pub(super) fn young_scavenge_cap_due() -> bool {
     from_space_in_use >= scavenge_nursery_cap_dueness_bytes()
 }
 
+/// #10169: does the young generation hold at least one BASE nursery cap of
+/// bytes — a minor's worth of data, whatever the adaptive cap currently says?
+/// Read by `string::json_leaf_prefers_arena` from inside a suppressed
+/// construction window. The base cap rather than [`young_scavenge_cap_due`]'s
+/// adaptive one on purpose: after a fully-live minor the adaptive cap scales
+/// past the very tree that made it scale, and a gate keyed on it would flip
+/// the next leaf back to the malloc registry, where one tracked leaf is enough
+/// to veto the untraced promotion the route exists to enable.
+pub(crate) fn young_generation_holds_a_nursery() -> bool {
+    nursery_cap_active()
+        && crate::arena::copying_from_space_in_use_bytes() >= gc_scavenge_nursery_cap_bytes()
+}
+
 /// The cap value [`young_scavenge_cap_due`] compares against.
 ///
 /// Split out only so a test can make the cap due without allocating the real
@@ -1085,6 +1098,9 @@ crate::perry_thread_local! {
     pub(super) static GC_DEFERRED_REQUEST: Cell<DeferredGcRequest> =
         const { Cell::new(DeferredGcRequest::None) };
     pub(super) static GC_OLD_RECLAIM_PENDING: Cell<bool> = const { Cell::new(false) };
+    /// #10169: a document-sized JSON leaf was born old under young pressure
+    /// since the last trigger decision (`note_young_leaf_born_old`).
+    pub(super) static GC_YOUNG_LEAF_BORN_OLD: Cell<bool> = const { Cell::new(false) };
     pub(super) static GC_LAST_OLD_RECLAIM_IN_USE_BYTES: Cell<usize> = const { Cell::new(0) };
     /// Live allocated arena bytes measured right after the last FULL
     /// mark-sweep — the baseline for major-GC pacing
@@ -3117,7 +3133,7 @@ struct BudgetedGcCycle {
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
-enum BudgetedGcTrigger {
+pub(super) enum BudgetedGcTrigger {
     OldReclaim,
     ArenaBytes,
     /// The young-generation scavenge cap ([`young_scavenge_cap_due`]).
@@ -3230,7 +3246,31 @@ pub(crate) fn trigger_path_hot_slot_indices() -> Vec<(&'static str, u32)> {
     ]
 }
 
-fn gc_budgeted_due_trigger() -> Option<BudgetedGcTrigger> {
+/// #10169: record that a document-sized JSON leaf was just born old in the
+/// arena because the young generation is at least that large and has not been
+/// measured as dying (`string::json_leaf_prefers_arena`). Read once by the
+/// next trigger decision.
+pub(crate) fn note_young_leaf_born_old() {
+    GC_YOUNG_LEAF_BORN_OLD.with(|flag| flag.set(true));
+}
+
+pub(super) fn gc_budgeted_due_trigger() -> Option<BudgetedGcTrigger> {
+    // #10169: a leaf born old under young pressure gives the nursery minor
+    // ONE-TIME priority over old-reclaim, and only while the young generation
+    // is still unmeasured. A young generation that a minor has already
+    // measured as retained wholesale is either promoted (so the next leaf
+    // finds it small) or, when it dies at every loop edge, best left to the
+    // old-reclaim full that sweeps it in Eden together with the leaf. The one
+    // case that must not fall through is an unmeasured young generation that
+    // stays live: old-reclaim would re-mark it in place at every full, so it
+    // is promoted by a minor first. The flag is consumed here whatever the
+    // decision, so it can never starve old-reclaim.
+    if GC_YOUNG_LEAF_BORN_OLD.with(Cell::get) {
+        GC_YOUNG_LEAF_BORN_OLD.with(|flag| flag.set(false));
+        if !super::young_generation_measured_retained() && young_scavenge_cap_due() {
+            return Some(BudgetedGcTrigger::YoungScavengeCap);
+        }
+    }
     let old_pending = GC_OLD_RECLAIM_PENDING.with(Cell::get);
     // #6010: external Map/Set side-buffer bytes escalate to OldReclaim too.
     let old_in_use =

crates/perry-runtime/src/gc/promote_in_place.rs

@@ -577,6 +577,27 @@ pub(crate) fn last_young_survival_permille() -> Option<u64> {
     LAST_YOUNG_SURVIVAL_PERMILLE.with(Cell::get)
 }
 
+/// #10169: did the previous copying minor measure the young generation as
+/// retained wholesale? This is the in-place promotion signal read directly,
+/// without the test-build opt-in that gates the promotion itself, so a
+/// scheduling decision elsewhere can key on the measurement alone. `None` —
+/// no copying minor has run on this thread — is deliberately `false`.
+pub(crate) fn young_generation_measured_retained() -> bool {
+    LAST_YOUNG_SURVIVAL_PERMILLE
+        .with(Cell::get)
+        .is_some_and(|permille| permille >= PROMOTE_SURVIVAL_THRESHOLD_PERMILLE)
+}
+
+/// #10169: the complement that is NOT `!measured_retained`: the previous
+/// copying minor measured the young generation as mostly garbage. `None` — no
+/// measurement yet — is `false` here too, so an unmeasured young generation
+/// is neither retained nor dying.
+pub(crate) fn young_generation_measured_dying() -> bool {
+    LAST_YOUNG_SURVIVAL_PERMILLE
+        .with(Cell::get)
+        .is_some_and(|permille| permille < PROMOTE_SURVIVAL_THRESHOLD_PERMILLE)
+}
+
 #[cfg(test)]
 pub(crate) fn promoted_dead_bytes_since_full() -> usize {
     PROMOTED_DEAD_BYTES.with(Cell::get)
@@ -644,12 +665,12 @@ impl Drop for InPlacePromotionTestGuard {
 /// MEASUREMENT of "almost nothing survived", and only this exercises the
 /// `None` arm of the decision.
 #[cfg(test)]
-pub(super) fn clear_young_survival_for_tests() {
+pub(crate) fn clear_young_survival_for_tests() {
     LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(None));
 }
 
 #[cfg(test)]
-pub(super) fn seed_young_survival_for_tests(permille: u64) {
+pub(crate) fn seed_young_survival_for_tests(permille: u64) {
     LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(Some(permille)));
 }
 

crates/perry-runtime/src/gc/tests/mod.rs

@@ -74,4 +74,5 @@ mod triggers;
 mod typed_layout_intact_residual;
 mod u8_inline_cache;
 mod weak_read_barrier;
+mod young_leaf_route;
 mod young_log_tests;

crates/perry-runtime/src/gc/tests/young_leaf_route.rs

@@ -0,0 +1,67 @@
+//! #10169: a document-sized JSON leaf born old under young pressure gives the
+//! nursery minor one-time priority over old-reclaim, and only while the young
+//! generation is unmeasured. Both halves are asserted: the priority fires
+//! exactly once per leaf, and a measured young generation buys none.
+
+use super::super::policy::{
+    gc_budgeted_due_trigger, note_young_leaf_born_old, BudgetedGcTrigger,
+    ScavengeNurseryCapTestGuard, GC_OLD_RECLAIM_PENDING,
+};
+use super::super::*;
+use super::support::*;
+
+#[test]
+fn young_leaf_born_old_prioritises_the_nursery_minor_until_measured() {
+    let _isolation = GcTestIsolationGuard::new();
+    let _pacing = crate::gc::policy::force_moving_gc_pacing();
+    let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    let _cap_due = ScavengeNurseryCapTestGuard::due_at_bytes(1);
+    // The isolated arena starts empty; one young allocation makes "due at one
+    // byte" actually due.
+    let filler = [b'y'; 64];
+    crate::string::js_string_from_bytes(filler.as_ptr(), filler.len() as u32);
+    assert!(
+        crate::arena::copying_from_space_in_use_bytes() >= 1,
+        "fixture: the young generation must hold something for the cap to be due"
+    );
+    let previous_survival = last_young_survival_permille();
+    GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(true));
+    assert_eq!(
+        gc_budgeted_due_trigger(),
+        Some(BudgetedGcTrigger::OldReclaim),
+        "fixture: old-reclaim must be due before the leaf can outrank it"
+    );
+
+    // Unmeasured young generation: the leaf buys the nursery minor exactly one
+    // decision, then old-reclaim is back.
+    clear_young_survival_for_tests();
+    note_young_leaf_born_old();
+    assert_eq!(
+        gc_budgeted_due_trigger(),
+        Some(BudgetedGcTrigger::YoungScavengeCap)
+    );
+    assert_eq!(
+        gc_budgeted_due_trigger(),
+        Some(BudgetedGcTrigger::OldReclaim)
+    );
+
+    // Measured as retained: no priority, and the flag is still consumed.
+    seed_young_survival_for_tests(999);
+    note_young_leaf_born_old();
+    assert_eq!(
+        gc_budgeted_due_trigger(),
+        Some(BudgetedGcTrigger::OldReclaim)
+    );
+    clear_young_survival_for_tests();
+    assert_eq!(
+        gc_budgeted_due_trigger(),
+        Some(BudgetedGcTrigger::OldReclaim),
+        "a consumed flag must not be honoured later"
+    );
+
+    GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(false));
+    match previous_survival {
+        Some(permille) => seed_young_survival_for_tests(permille),
+        None => clear_young_survival_for_tests(),
+    }
+}

crates/perry-runtime/src/json/mod.rs

@@ -600,13 +600,15 @@ pub(crate) fn json_string_from_native_output_bytes(bytes: &[u8]) -> *mut StringH
         crate::string::compute_utf16_len(bytes.as_ptr(), len)
     };
     stringify_flat::service_json_output_sweep_boundary();
-    let (ptr, data) = crate::string::json_output_storage_alloc(len);
+    let (ptr, data, malloc_tracked) = crate::string::json_output_storage_alloc(len);
     unsafe {
         crate::string::init_string_header(ptr, utf16_len, len, len, 0, 0);
         // GC_STORE_AUDIT(POINTER_FREE): completed JSON payload bytes.
         std::ptr::copy_nonoverlapping(bytes.as_ptr(), data, len as usize);
     }
-    stringify_flat::note_completed_malloc_json_output(len);
+    if malloc_tracked {
+        stringify_flat::note_completed_malloc_json_output(len);
+    }
     ptr
 }
 

crates/perry-runtime/src/json/stringify_flat.rs

@@ -416,7 +416,7 @@ unsafe fn emit_two_field_parsed_string_object(
 
     let large_output = bytes >= JSON_MALLOC_OUTPUT_THRESHOLD;
     let construction = large_output.then(crate::gc::GcSuppressScope::new);
-    let (result, output) = json_output_storage_alloc(bytes);
+    let (result, output, malloc_tracked) = json_output_storage_alloc(bytes);
     let value = input.with_const_ptr(|obj: *const crate::ObjectHeader| {
         let keys = crate::object::object_keys_array(obj);
         init_string_header(result, units, bytes, bytes, 0, 0);
@@ -449,7 +449,7 @@ unsafe fn emit_two_field_parsed_string_object(
         Some(JSValue::string_ptr(result))
     });
     drop(construction);
-    if large_output {
+    if malloc_tracked {
         note_completed_malloc_json_output(bytes);
     }
     value

crates/perry-runtime/src/json/stringify_record_output.rs

@@ -460,7 +460,7 @@ unsafe fn emit_cached_record_uncached(
         return None;
     }
     let construction = large_output.then(crate::gc::GcSuppressScope::new);
-    let (result, output) = json_output_storage_alloc(bytes);
+    let (result, output, malloc_tracked) = json_output_storage_alloc(bytes);
     init_string_header(result, units, bytes, bytes, 0, 0);
     let value = input.with_const_ptr(|obj: *const crate::ObjectHeader| {
         let mut at = 0usize;
@@ -505,7 +505,7 @@ unsafe fn emit_cached_record_uncached(
         Some(JSValue::string_ptr(result))
     });
     drop(construction);
-    if large_output {
+    if malloc_tracked {
         super::stringify_flat::note_completed_malloc_json_output(bytes);
     }
     value
@@ -562,7 +562,7 @@ unsafe fn emit_cached_record_memo(
     }
     let repeated_candidate = bytes as usize <= MAX_REPEATED_OUTPUT_BYTES;
     let construction = large_output.then(crate::gc::GcSuppressScope::new);
-    let (result, output) = json_output_storage_alloc(bytes);
+    let (result, output, malloc_tracked) = json_output_storage_alloc(bytes);
     init_string_header(result, units, bytes, bytes, 0, 0);
     let value = input.with_const_ptr(|obj: *const crate::ObjectHeader| {
         let mut at = 0usize;
@@ -665,7 +665,7 @@ unsafe fn emit_cached_record_memo(
         Some(JSValue::string_ptr(result))
     });
     drop(construction);
-    if large_output {
+    if malloc_tracked {
         super::stringify_flat::note_completed_malloc_json_output(bytes);
     }
     value
@@ -745,7 +745,7 @@ unsafe fn emit_record(obj: *const crate::ObjectHeader, fields: usize) -> Option<
         return None;
     }
     let construction = large_output.then(crate::gc::GcSuppressScope::new);
-    let (result, output) = json_output_storage_alloc(bytes);
+    let (result, output, malloc_tracked) = json_output_storage_alloc(bytes);
     init_string_header(result, units, bytes, bytes, 0, 0);
     let value = input.with_const_ptr(|obj: *const crate::ObjectHeader| {
         let keys = crate::object::object_keys_array(obj);
@@ -797,7 +797,7 @@ unsafe fn emit_record(obj: *const crate::ObjectHeader, fields: usize) -> Option<
         Some(JSValue::string_ptr(result))
     });
     drop(construction);
-    if large_output {
+    if malloc_tracked {
         super::stringify_flat::note_completed_malloc_json_output(bytes);
     }
     value

crates/perry-runtime/src/json/stringify_string_tests.rs

@@ -98,3 +98,63 @@ fn direct_quoted_raw_strings_preserve_fallback_length_semantics() {
         }
     }
 }
+
+/// #10169: a large leaf stays malloc-tracked while the young generation is
+/// smaller than the leaf or was measured as mostly garbage; it is born old in
+/// the arena when the young generation holds at least the leaf's own bytes
+/// and is unmeasured or measured as retained — the shape of a freshly parsed
+/// document whose result the caller is about to stringify.
+#[test]
+fn large_json_leaf_routes_by_young_generation_occupancy() {
+    if !crate::gc::gen_gc_enabled() {
+        return;
+    }
+    let previous_survival = crate::gc::last_young_survival_permille();
+    let _suppress = crate::gc::GcSuppressScope::new();
+    let young_before = crate::arena::copying_from_space_in_use_bytes();
+    let leaf = (young_before as u32 + (1 << 20)).max(crate::string::JSON_MALLOC_OUTPUT_THRESHOLD);
+    crate::gc::seed_young_survival_for_tests(999);
+    let (_, _, tracked_below) = crate::string::json_output_storage_alloc(leaf);
+    assert!(
+        tracked_below,
+        "a young generation smaller than the leaf keeps malloc tracking"
+    );
+
+    let filler = vec![b'y'; 1024];
+    while crate::arena::copying_from_space_in_use_bytes() < leaf as usize {
+        crate::string::js_string_from_bytes(filler.as_ptr(), filler.len() as u32);
+    }
+    if !crate::gc::young_generation_holds_a_nursery() {
+        let (_, _, tracked_below_nursery) = crate::string::json_output_storage_alloc(leaf);
+        assert!(
+            tracked_below_nursery,
+            "a young generation below one nursery keeps malloc tracking"
+        );
+        while !crate::gc::young_generation_holds_a_nursery() {
+            crate::string::js_string_from_bytes(filler.as_ptr(), filler.len() as u32);
+        }
+    }
+    crate::gc::clear_young_survival_for_tests();
+    let (_, _, tracked_unmeasured) = crate::string::json_output_storage_alloc(leaf);
+    assert!(
+        !tracked_unmeasured,
+        "an unmeasured young generation at or above the leaf size births the leaf in the arena"
+    );
+    crate::gc::seed_young_survival_for_tests(100);
+    let (_, _, tracked_dying) = crate::string::json_output_storage_alloc(leaf);
+    assert!(
+        tracked_dying,
+        "a young generation measured as mostly garbage keeps malloc tracking"
+    );
+    crate::gc::seed_young_survival_for_tests(999);
+    let (_, _, tracked_retained) = crate::string::json_output_storage_alloc(leaf);
+    assert!(
+        !tracked_retained,
+        "a retained young generation at or above the leaf size births the leaf in the arena"
+    );
+
+    match previous_survival {
+        Some(permille) => crate::gc::seed_young_survival_for_tests(permille),
+        None => crate::gc::clear_young_survival_for_tests(),
+    }
+}

crates/perry-runtime/src/string/json_construction.rs

@@ -23,7 +23,8 @@ pub(crate) unsafe fn string_from_json_bytes(
     };
     let (header, data) = if raw.is_null() {
         if large_json_leaf {
-            json_output_storage_alloc(len)
+            let (header, data, _malloc_tracked) = json_output_storage_alloc(len);
+            (header, data)
         } else {
             string_storage_alloc(len)
         }