From 7cee64ad73cd761385c53f9901142cfe21205d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 13:16:56 +0200 Subject: [PATCH 01/10] codegen: collapse the class-field GET tower to one exit The monomorphic `this.field` / `obj.field` tower in expr/property_get.rs emitted four runtime call sites per site -- `js_typed_feedback_class_field_get_guard` when the #5093 inline pre-check missed, `js_throw_type_error_property_access` for a nullish receiver, and one or two `js_object_get_field_by_name_f64` by-name lookups -- spread over class_field_get.{fast,fallback,merge,throw_nullish,fallback_lookup}. In @babel/parser that is 13,636 sites and 27 % of the module's IR, and because every call is a statepoint the .perry_gcmap scaled with it too. Keep the pre-check and the fast slot load byte-identical, and replace the four miss arms with ONE call to `js_class_field_get_ic` -- the runtime helper the #5391 path-2 full outline a few lines above already calls. It runs the same guard, reads the same slot at the same header-relative offset on a pass, throws the same nullish TypeError (#7153 put that check there for exactly this equivalence) and falls back to `js_object_get_field_by_name_f64`, recording the fallback itself under the same `typed_feedback_enabled()` gate codegen used. A plain number is self-boxing under NaN-boxing, so a `requires_raw_f64` site reads the helper's return exactly as the old phi read the by-name fallback's -- neither arm converted. The key-handle load/bitcast/mask also sinks into the cold miss block, its only remaining consumer. Measured (fixture with one boxed field, two raw-f64 fields, a read on a bare `Named` parameter, reads on `this`, and a raw-f64 read in a loop): * per site: 88 -> 47 IR instructions, 3 runtime calls -> 1, 7 -> 4 blocks; * the `class_field_inline.deref` pre-check is instruction-for-instruction identical (SSA numbering shifts by 3 because the entry block lost the sunk key load); * the `class_field_get.fast` block LOSES its two RS4GC relocation reloads (14 instructions, 2 `call i64 asm "gc-leaf-function"`): it used to be reachable from the guard call, i.e. across a statepoint, and is now reachable only from the pre-check, which crosses none. The inline hit path is strictly shorter than before; * @babel/parser .text 17,640,473 -> 16,389,647 (-7.09 %), .perry_gcmap 809,047 -> 792,749 (-2.01 %), O0-fallback units unchanged. Runtime behaviour is unchanged: on benchmarks/tls-budget/interp.ts the typed-feedback trace reports the same 273 sites with identical guard_passes (817,155,616), guard_failures (202,359,184) and fallback_calls (202,359,184), zero per-site differences, and identical program output. The raw-f64 site's dynamic-fallback native-value record now names `js_class_field_get_ic`; that pair is ADDED to verify/raw_f64.rs rather than replacing the by-name pair, which verify/tests.rs still pins. Tests: a new cargo-test-visible codegen module pins the one-exit shape (the pre-check branches INTO the fast load, the miss arm is exactly one call, the three retired blocks are gone, and the three retired symbols are absent as CALL FORMS -- a bare substring is satisfied by the `declare` line alone). Four runtime tests cover `js_class_field_get_ic`'s arms, which had none. Five existing assertions that used the retired symbols as the proxy for "a guarded class-field read was emitted" are re-pointed: the positives accept either witness (other lowerings still call the old symbols), and the negatives gain the IC as an excluded symbol so they cannot pass on a still-guarded body. (cherry picked from commit cf7c6caed7adfa4376c3bacc68ccb29724be6f76) --- .../src/codegen/argument_shape_clone_tests.rs | 10 +- .../src/expr/array_callback_shape_tests.rs | 8 +- .../src/expr/class_field_get_shape_tests.rs | 310 ++++++++++++++++++ crates/perry-codegen/src/expr/mod.rs | 2 + crates/perry-codegen/src/expr/property_get.rs | 217 ++++++------ .../src/native_value/verify/raw_f64.rs | 7 + .../src/stmt/element_shape_loop_tests.rs | 1 + .../tests/native_proof_regressions.rs | 9 +- crates/perry-codegen/tests/typed_feedback.rs | 44 ++- .../perry-runtime/src/typed_feedback/tests.rs | 145 ++++++++ .../tests/issue_8774_argument_shape_clones.rs | 3 +- 11 files changed, 621 insertions(+), 135 deletions(-) create mode 100644 crates/perry-codegen/src/expr/class_field_get_shape_tests.rs diff --git a/crates/perry-codegen/src/codegen/argument_shape_clone_tests.rs b/crates/perry-codegen/src/codegen/argument_shape_clone_tests.rs index d2bd3ad63f..92d05f7a9b 100644 --- a/crates/perry-codegen/src/codegen/argument_shape_clone_tests.rs +++ b/crates/perry-codegen/src/codegen/argument_shape_clone_tests.rs @@ -210,13 +210,20 @@ fn guarded_call_routes_to_shadow_rooted_direct_field_clone() { clone.contains("inttoptr i64") && clone.contains("getelementptr double"), "the clone must use direct declared-field addressing:\n{clone}" ); + // One-exit class-field GET: the tower's miss arm is a single + // `js_class_field_get_ic` call whose body IS the guard, so the IC is now a + // witness of "a field IC diamond was rebuilt" and must be excluded here + // too -- otherwise this negative assertion silently stops covering the + // shape it was written for. assert!( !clone.contains("js_typed_feedback_class_field_get_guard") + && !clone.contains("js_class_field_get_ic") && !clone.contains("shape_descriptor_by_id"), "the clone fast body must not rebuild the field IC diamond:\n{clone}" ); assert!( - generic.contains("js_typed_feedback_class_field_get_guard") + generic.contains("js_class_field_get_ic") + || generic.contains("js_typed_feedback_class_field_get_guard") || generic.contains("js_object_get_field"), "the generic fallback must retain guarded field semantics:\n{generic}" ); @@ -264,6 +271,7 @@ fn unannotated_parameter_uses_the_runtime_validated_class_overlay() { ); assert!( !clone.contains("js_typed_feedback_class_field_get_guard") + && !clone.contains("js_class_field_get_ic") && !clone.contains("shape_descriptor_by_id"), "the unannotated clone must not rebuild the field IC diamond:\n{clone}" ); diff --git a/crates/perry-codegen/src/expr/array_callback_shape_tests.rs b/crates/perry-codegen/src/expr/array_callback_shape_tests.rs index 5258bb195b..13ad7f81ca 100644 --- a/crates/perry-codegen/src/expr/array_callback_shape_tests.rs +++ b/crates/perry-codegen/src/expr/array_callback_shape_tests.rs @@ -150,8 +150,13 @@ fn callback_body(ir: &str) -> &str { fn inline_array_callback_field_read_has_no_shape_guard() { let ir = emit(&module_with_callback(false)); let callback = callback_body(&ir); + // `js_class_field_get_ic` is the one-exit tower's miss arm (guard + slot + // load + by-name fallback in one call), so it is the current witness that a + // guarded read was emitted; without it this negative assertion would pass + // on a callback that is still fully guarded. assert!( !callback.contains("js_typed_feedback_class_field_get_guard") + && !callback.contains("js_class_field_get_ic") && !callback.contains("js_object_get_field_by_name_f64"), "the proven element parameter must use a direct fixed-offset load:\n{callback}" ); @@ -166,7 +171,8 @@ fn callback_source_array_alias_keeps_the_shape_guard() { let ir = emit(&module_with_callback(true)); let callback = callback_body(&ir); assert!( - callback.contains("js_typed_feedback_class_field_get_guard") + callback.contains("js_class_field_get_ic") + || callback.contains("js_typed_feedback_class_field_get_guard") || callback.contains("js_object_get_field_by_name_f64"), "declaring the source-array argument must deny the cross-boundary fact:\n{callback}" ); diff --git a/crates/perry-codegen/src/expr/class_field_get_shape_tests.rs b/crates/perry-codegen/src/expr/class_field_get_shape_tests.rs new file mode 100644 index 0000000000..d07a664bae --- /dev/null +++ b/crates/perry-codegen/src/expr/class_field_get_shape_tests.rs @@ -0,0 +1,310 @@ +//! The monomorphic class-field GET tower has ONE exit. +//! +//! The tower in [`super::property_get`] used to emit four runtime call sites +//! per `this.field` / `obj.field` read — `js_typed_feedback_class_field_get_ +//! guard` when the #5093 inline pre-check missed, `js_throw_type_error_ +//! property_access` for a nullish receiver, and one or two +//! `js_object_get_field_by_name_f64` by-name lookups — spread over +//! `class_field_get.{fast,fallback,merge,throw_nullish,fallback_lookup}`. At +//! ~74 pre-RS4GC instructions and 13,636 sites in @babel/parser that arm alone +//! was 27 % of the module's IR, and because every call is a statepoint the +//! `.perry_gcmap` grew with it. +//! +//! Everything behind the pre-check now lives in `js_class_field_get_ic`, the +//! runtime helper the #5391 path-2 full outline already called. What must hold, +//! and why a label-presence check cannot show it: +//! +//! 1. **The inline hit path is still there and still REACHED.** A one-exit +//! tower that also lost its pre-check would be a pure pessimisation, and +//! every assertion below about "one call" would still pass. So: a `cond_br` +//! out of `class_field_inline.deref` INTO `class_field_get.fast`, and a +//! `load double` inside that block. (CLAUDE.md, "a gate must assert its +//! subject was live".) +//! 2. **The miss arm is exactly one call.** Counted, not merely "the IC is +//! mentioned": a regression that re-adds the by-name fallback beside the IC +//! would keep every positive assertion true. +//! 3. **The retired symbols are not CALLED at this site.** Asserted as +//! `call @name(`, never as a bare substring — all three are still +//! declared in the module (and still emitted by other towers), so a +//! substring test is satisfied by the `declare` line alone and can never +//! fail. That exact trap is why the older assertions in +//! `tests/typed_feedback.rs` were rewritten rather than deleted. +//! 4. **Both representations.** A `number` field (`require_raw_f64 = 1`) and an +//! `any` field (boxed) take the same shape; the raw-f64 site must not grow a +//! second by-name call back. + +use crate::compile_module; +use perry_hir::types::Type; +use perry_hir::{Class, ClassField, Expr, Function, Module, ModuleInitKind, Param, Stmt}; + +const PARAM_ID: u32 = 3; + +/// Blocks the tower is allowed to create, in the order it creates them. +const TOWER_BLOCKS: [&str; 4] = [ + "class_field_inline.deref", + "class_field_inline.guardcall", + "class_field_get.fast", + "class_field_get.merge", +]; + +/// Blocks the four-exit tower created that must no longer exist. +const RETIRED_BLOCKS: [&str; 3] = [ + "class_field_get.fallback", + "class_field_get.throw_nullish", + "class_field_get.fallback_lookup", +]; + +/// Call FORMS (not bare names) for the runtime entries the tower stopped +/// emitting. Each is still declared in every module and still emitted by other +/// lowerings, so only the call form can distinguish "not called here". +const RETIRED_CALLS: [&str; 3] = [ + "call i32 @js_typed_feedback_class_field_get_guard(", + "call void @js_throw_type_error_property_access(", + "call double @js_object_get_field_by_name_f64(", +]; + +fn field(name: &str, ty: Type) -> ClassField { + ClassField { + name: name.to_string(), + key_expr: None, + ty, + init: None, + is_private: false, + is_readonly: false, + decorators: Vec::new(), + } +} + +fn point_class(field_ty: Type) -> Class { + Class { + id: 101, + name: "Point".to_string(), + type_params: Vec::new(), + extends: None, + extends_name: None, + native_extends: None, + extends_expr: None, + heritage_lexically_shadowed: false, + fields: vec![field("x", field_ty)], + constructor: None, + methods: Vec::new(), + getters: Vec::new(), + setters: Vec::new(), + static_accessor_names: Vec::new(), + static_accessor_fn_ids: Vec::new(), + computed_members: Vec::new(), + static_fields: Vec::new(), + static_methods: Vec::new(), + decorators: Vec::new(), + is_exported: false, + aliases: Vec::new(), + is_nested: false, + alloc_width_hint: 0, + specialized_from: None, + } +} + +/// `function probe(p: Point) { return p.x }` — a bare `Named` parameter, which +/// is what routes to the generic `class_field_get.*` tower (#8033: the +/// numeric-specific `class_field_get_number.*` path needs a +/// `stable_local_type_proof` a parameter has not got). +fn probe_module(field_ty: Type) -> Module { + let mut m = Module::new("class_field_get_shape.ts"); + m.classes = vec![point_class(field_ty)]; + m.functions = vec![Function { + id: 1, + name: "probe".to_string(), + type_params: Vec::new(), + params: vec![Param { + id: PARAM_ID, + name: "p".to_string(), + ty: Type::Named("Point".to_string()), + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }], + return_type: Type::Any, + body: vec![Stmt::Return(Some(Expr::PropertyGet { + byte_offset: 0, + object: Box::new(Expr::LocalGet(PARAM_ID)), + property: "x".to_string(), + }))], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }]; + m.init_kind = ModuleInitKind::Eager; + m +} + +fn ir(field_ty: Type) -> String { + String::from_utf8( + compile_module( + &probe_module(field_ty), + super::class_field_barrier_tests::ir_opts(), + ) + .expect("module compiles"), + ) + .expect("LLVM IR should be UTF-8") +} + +/// Body of the first block whose label line starts with `prefix`. Block labels +/// carry per-function numeric suffixes (`.fast.6`), so callers pass the stable +/// prefix; a label line is line-initial and ends in `:`, while branches and +/// phis that merely mention it are indented. +fn block_body<'a>(ir: &'a str, prefix: &str) -> Option<&'a str> { + let needle = format!("\n{prefix}"); + let mut from = 0; + while let Some(rel) = ir[from..].find(&needle) { + let label_start = from + rel + 1; + let line_end = label_start + ir[label_start..].find('\n')?; + if ir[label_start..line_end].ends_with(':') { + let rest = &ir[line_end + 1..]; + let end = match (rest.find("\n\n"), rest.find("\n}")) { + (Some(a), Some(b)) => a.min(b), + (a, b) => a.or(b).unwrap_or(rest.len()), + }; + return Some(&rest[..end]); + } + from = line_end; + } + None +} + +/// The `prefix.N:` label as rendered, e.g. `class_field_get.fast.6`. +fn block_label(ir: &str, prefix: &str) -> Option { + let needle = format!("\n{prefix}"); + let mut from = 0; + while let Some(rel) = ir[from..].find(&needle) { + let label_start = from + rel + 1; + let line_end = label_start + ir[label_start..].find('\n')?; + let line = &ir[label_start..line_end]; + if line.ends_with(':') { + return Some(line.trim_end_matches(':').to_string()); + } + from = line_end; + } + None +} + +fn assert_one_exit(field_ty: Type, what: &str) { + let ir = ir(field_ty); + + // (1) the pre-check exists and BRANCHES INTO the fast slot load. + let deref = block_body(&ir, "class_field_inline.deref") + .unwrap_or_else(|| panic!("{what}: no class_field_inline.deref block:\n{ir}")); + let fast_label = block_label(&ir, "class_field_get.fast") + .unwrap_or_else(|| panic!("{what}: no class_field_get.fast block:\n{ir}")); + assert!( + deref.contains("br i1 ") && deref.contains(&format!("label %{fast_label}")), + "{what}: class_field_inline.deref does not branch into {fast_label}; \ + the inline hit path is dead:\n{deref}" + ); + let fast = block_body(&ir, "class_field_get.fast") + .unwrap_or_else(|| panic!("{what}: no class_field_get.fast body:\n{ir}")); + assert!( + fast.contains("load double"), + "{what}: the fast block no longer loads the slot:\n{fast}" + ); + assert!( + !fast.contains("call "), + "{what}: the fast block must stay call-free:\n{fast}" + ); + + // (2) the miss arm is EXACTLY one call. + let guardcall = block_body(&ir, "class_field_inline.guardcall") + .unwrap_or_else(|| panic!("{what}: no class_field_inline.guardcall block:\n{ir}")); + let calls = guardcall.matches("call ").count(); + assert_eq!( + calls, 1, + "{what}: the miss arm must be ONE call, found {calls}:\n{guardcall}" + ); + assert!( + guardcall.contains("call double @js_class_field_get_ic("), + "{what}: the miss arm's one call is not js_class_field_get_ic:\n{guardcall}" + ); + + // (3) the retired blocks and the retired CALL FORMS are gone. + for block in RETIRED_BLOCKS { + assert!( + block_body(&ir, block).is_none(), + "{what}: retired block {block} is still emitted:\n{ir}" + ); + } + for call in RETIRED_CALLS { + assert!( + !ir.contains(call), + "{what}: retired call form `{call}` is still emitted:\n{ir}" + ); + } + + // (4) the merge is a two-way phi (fast value, IC value) and nothing else. + let merge = block_body(&ir, "class_field_get.merge") + .unwrap_or_else(|| panic!("{what}: no class_field_get.merge block:\n{ir}")); + let phis: Vec<&str> = merge.lines().filter(|l| l.contains("phi double")).collect(); + assert_eq!(phis.len(), 1, "{what}: merge is not a single phi:\n{merge}"); + assert_eq!( + phis[0].matches('[').count(), + 2, + "{what}: merge phi does not have exactly two incoming edges (fast, IC):\n{}", + phis[0] + ); + + // (5) the tower creates exactly four blocks. + for block in TOWER_BLOCKS { + assert!( + block_body(&ir, block).is_some(), + "{what}: tower block {block} missing:\n{ir}" + ); + } +} + +#[test] +fn boxed_class_field_get_has_one_exit() { + assert_one_exit(Type::Any, "any-typed field"); +} + +#[test] +fn raw_f64_class_field_get_has_one_exit() { + // `number` makes `requires_raw_f64` true, which used to emit a SECOND + // by-name fallback record/call pair. The IC takes `require_raw_f64` as an + // argument instead, so the site stays at one call. + assert_one_exit(Type::Number, "number-typed field"); +} + +#[test] +fn the_ic_call_carries_the_guard_operands() { + // The helper runs the same guard the inline miss arm used to call, so it + // must receive the same seven operands in the same order — site id, + // receiver, expected class id, expected shape id, key, field index, + // require_raw_f64. A dropped operand compiles and silently guards on the + // wrong thing. + let ir = ir(Type::Number); + let guardcall = block_body(&ir, "class_field_inline.guardcall") + .expect("guardcall block") + .to_string(); + let line = guardcall + .lines() + .find(|l| l.contains("@js_class_field_get_ic(")) + .expect("the IC call line"); + let args = line + .rsplit_once("@js_class_field_get_ic(") + .expect("call args") + .1; + let args = args.rsplit_once(')').expect("closing paren").0; + let tys: Vec<&str> = args + .split(", ") + .map(|a| a.split_whitespace().next().unwrap_or("")) + .collect(); + assert_eq!( + tys, + vec!["i64", "double", "i32", "i32", "i64", "i32", "i32"], + "IC call signature drifted from the guard's operand list:\n{line}" + ); +} diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index a0df121336..9c251f68ca 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -156,6 +156,8 @@ mod array_push_guard_tests; mod barrier_stem_census_tests; #[cfg(test)] mod class_field_barrier_tests; +#[cfg(test)] +mod class_field_get_shape_tests; mod dispatch; #[cfg(test)] mod index_set_barrier_tests; diff --git a/crates/perry-codegen/src/expr/property_get.rs b/crates/perry-codegen/src/expr/property_get.rs index b3b5b32c1c..51fe3d1214 100644 --- a/crates/perry-codegen/src/expr/property_get.rs +++ b/crates/perry-codegen/src/expr/property_get.rs @@ -1660,29 +1660,24 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { ); return Ok(val); } - // #5093: build the guard operands once, up front, so both - // the inline shape pre-check and the guard-call fallback - // can reference them. - let (obj_bits, obj_handle, key_raw) = { + // #5093: build the guard operands once, up front, so + // both the inline shape pre-check and the fast slot + // load can reference them. + let (obj_bits, obj_handle) = { let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(&recv_box); let obj_handle = blk.and(I64, &obj_bits, POINTER_MASK_I64); - let key_box = blk.load(DOUBLE, &key_handle_global); - let key_bits = blk.bitcast_double_to_i64(&key_box); - let key_raw = blk.and(I64, &key_bits, POINTER_MASK_I64); - (obj_bits, obj_handle, key_raw) + (obj_bits, obj_handle) }; let fast_idx = ctx.new_block("class_field_get.fast"); - let fallback_idx = ctx.new_block("class_field_get.fallback"); let merge_idx = ctx.new_block("class_field_get.merge"); let fast_label = ctx.block_label(fast_idx); - let fallback_label = ctx.block_label(fallback_idx); let merge_label = ctx.block_label(merge_idx); // #5093: inline shape pre-check. On a monomorphic hit it // branches straight to the fast slot load, skipping the // cross-crate guard call; on a miss it leaves the current - // block at the guard-call path below (unchanged). + // block at the single-exit IC call below. let subclass_arms = crate::expr::class_field_inline_guard::class_field_subclass_arms( ctx, @@ -1703,9 +1698,48 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { &fast_label, &subclass_arms, ); - let guard_ok = ctx.block().call( - I32, - "js_typed_feedback_class_field_get_guard", + // ONE EXIT. Everything the pre-check could not prove — + // the guard call, the guard-PASS slot load, the nullish + // TypeError, the fallback record and the by-name lookup — + // is the body of `js_class_field_get_ic`, the same helper + // the #5391 path-2 full outline a few lines above already + // calls. Emitting the four arms inline cost ~74 IR + // instructions and 4 runtime call sites per monomorphic + // `this.field` read; @babel/parser has 13,636 of them + // (27 % of its IR), and each call site is a statepoint, + // so `.perry_gcmap` scaled with them too. + // + // The pre-check and the fast slot load below are + // unchanged, so a pre-check HIT costs exactly what it + // cost before (no call at all) and a pre-check MISS costs + // exactly one call — what the guard call alone already + // cost. Arm for arm the helper reproduces the diamond: + // guard PASS -> reads the same slot at the same + // header-relative offset; + // nullish -> throws the same + // `js_throw_type_error_property_access` + // (#7153 added that check to the helper + // for exactly this equivalence); + // guard FAIL -> records the fallback, then + // `js_object_get_field_by_name_f64`. + // A plain number is self-boxing under NaN-boxing, so a + // `requires_raw_f64` site reads the helper's return the + // same way the old phi read the by-name fallback's return + // — neither arm converted, and neither does this one. + // + // The key handle is loaded HERE, in the cold miss block, + // instead of in the entry block: the call is its only + // consumer, so sinking it takes a load, a bitcast and a + // mask off the inline hit path. + let key_raw = { + let blk = ctx.block(); + let key_box = blk.load(DOUBLE, &key_handle_global); + let key_bits = blk.bitcast_double_to_i64(&key_box); + blk.and(I64, &key_bits, POINTER_MASK_I64) + }; + let val_ic = ctx.block().call( + DOUBLE, + "js_class_field_get_ic", &[ (I64, &site_id), (DOUBLE, &recv_box), @@ -1716,9 +1750,57 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { (I32, requires_raw_f64_str), ], ); - let guard_pass = ctx.block().icmp_ne(I32, &guard_ok, "0"); - ctx.block() - .cond_br(&guard_pass, &fast_label, &fallback_label); + let ic_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + if requires_raw_f64 { + // The miss arm's value now arrives from the outlined + // IC instead of the inline by-name call, so the + // dynamic-fallback record follows it. Same contract + // as before — a JS value, not a proven raw double, + // with the raw-f64 layout rejected and invalidated by + // a runtime API (`native_value/verify/raw_f64.rs` + // enforces all three on this consumer). + let fallback = LoweredValue { + semantic: SemanticKind::JsValue, + rep: NativeRep::JsValue, + llvm_ty: DOUBLE, + value: val_ic.clone(), + }; + ctx.record_lowered_value_with_access_mode_and_facts( + "ClassFieldGet", + None, + "js_class_field_get_ic", + &fallback, + Some(BoundsState::Unknown), + None, + Some(BufferAccessMode::DynamicFallback), + Some(MaterializationReason::RuntimeApi), + None, + None, + Vec::new(), + vec![ + raw_f64_layout_fact( + None, + "rejected", + "class_field_get_guard", + Some(MaterializationReason::RuntimeApi), + ), + raw_f64_layout_fact( + None, + "invalidated", + "runtime_api", + Some(MaterializationReason::RuntimeApi), + ), + ], + false, + false, + vec![ + format!("class={}", class_name), + format!("field={}", property), + format!("field_index={}", field_idx_str), + ], + ); + } ctx.current_block = fast_idx; // arm64_32 watchOS: the object fields region begins at @@ -1782,109 +1864,10 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { ); } - ctx.current_block = fallback_idx; - // #7153: the guard rejects a nullish receiver along with - // every other shape miss, but a nullish field read must - // throw TypeError per spec — the by-name lookup below - // answers `undefined` and the program keeps running on a - // silent wrong value. Mirror the generic path's check - // (generic_dispatch.rs); the cost sits on the cold - // fallback arm only. - let (is_null, is_nullish) = { - let blk = ctx.block(); - let is_undef = - blk.icmp_eq(I64, &obj_bits, crate::nanbox::TAG_UNDEFINED_I64); - let is_null = blk.icmp_eq(I64, &obj_bits, crate::nanbox::TAG_NULL_I64); - let is_nullish = blk.or(I1, &is_undef, &is_null); - (is_null, is_nullish) - }; - let throw_idx = ctx.new_block("class_field_get.throw_nullish"); - let lookup_idx = ctx.new_block("class_field_get.fallback_lookup"); - let throw_label = ctx.block_label(throw_idx); - let lookup_label = ctx.block_label(lookup_idx); - ctx.block() - .cond_br(&is_nullish, &throw_label, &lookup_label); - - ctx.current_block = throw_idx; - let prop_entry = ctx.strings.entry(key_idx); - let prop_bytes_global = format!("@{}", prop_entry.bytes_global); - let prop_len_str = prop_entry.byte_len.to_string(); - let is_null_i32 = ctx.block().zext(I1, &is_null, I32); - ctx.block().call_void( - "js_throw_type_error_property_access", - &[ - (I32, &is_null_i32), - (PTR, &prop_bytes_global), - (I64, &prop_len_str), - ], - ); - ctx.block().unreachable(); - - ctx.current_block = lookup_idx; - let blk = ctx.block(); - crate::expr::emit_typed_feedback_record_call( - blk, - "js_typed_feedback_record_fallback_call", - &[(I64, &site_id)], - ); - let val_fallback_js = blk.call( - DOUBLE, - "js_object_get_field_by_name_f64", - &[(I64, &obj_bits), (I64, &key_raw)], - ); - let val_fallback = val_fallback_js.clone(); - let fallback_end_label = blk.label.clone(); - blk.br(&merge_label); - if requires_raw_f64 { - let fallback = LoweredValue { - semantic: SemanticKind::JsValue, - rep: NativeRep::JsValue, - llvm_ty: DOUBLE, - value: val_fallback_js.clone(), - }; - ctx.record_lowered_value_with_access_mode_and_facts( - "ClassFieldGet", - None, - "js_object_get_field_by_name_f64", - &fallback, - Some(BoundsState::Unknown), - None, - Some(BufferAccessMode::DynamicFallback), - Some(MaterializationReason::RuntimeApi), - None, - None, - Vec::new(), - vec![ - raw_f64_layout_fact( - None, - "rejected", - "class_field_get_guard", - Some(MaterializationReason::RuntimeApi), - ), - raw_f64_layout_fact( - None, - "invalidated", - "runtime_api", - Some(MaterializationReason::RuntimeApi), - ), - ], - false, - false, - vec![ - format!("class={}", class_name), - format!("field={}", property), - format!("field_index={}", field_idx_str), - ], - ); - } - ctx.current_block = merge_idx; return Ok(ctx.block().phi( DOUBLE, - &[ - (&val_fast, &fast_end_label), - (&val_fallback, &fallback_end_label), - ], + &[(&val_fast, &fast_end_label), (&val_ic, &ic_end_label)], )); } } diff --git a/crates/perry-codegen/src/native_value/verify/raw_f64.rs b/crates/perry-codegen/src/native_value/verify/raw_f64.rs index 51d2644550..0e15791933 100644 --- a/crates/perry-codegen/src/native_value/verify/raw_f64.rs +++ b/crates/perry-codegen/src/native_value/verify/raw_f64.rs @@ -96,6 +96,13 @@ pub(crate) fn raw_f64_dynamic_fallback_record(record: &NativeRepRecord) -> bool | ("PackedI32LoopGuard", "packed_i32_loop_fallback") | ("PackedU32LoopGuard", "packed_u32_loop_fallback") | ("ClassFieldGet", "js_object_get_field_by_name_f64") + // T3 one-exit class-field GET: the inline tower's miss arm is a + // single `js_class_field_get_ic` call instead of the nullish + // diamond + by-name lookup, so the dynamic-fallback record now + // names the IC. The by-name pair stays: it is still the contract + // asserted by `verify/tests.rs`, and dropping an entry rather + // than adding to it is exactly how #8858 silently lost one. + | ("ClassFieldGet", "js_class_field_get_ic") | ("ClassFieldSet", "js_object_set_field_by_name") ) } diff --git a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs index c78cfc72e5..b331ddd133 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs @@ -1488,6 +1488,7 @@ fn assert_clone_fires_call_free(ir: &str, what: &str) { assert!( !fast.contains("js_object_get_field_by_name_f64") && !fast.contains("js_typed_feedback_class_field_get_guard") + && !fast.contains("js_class_field_get_ic") && !fast.contains("js_number_coerce"), "{what}: the by-name field diamond must be gone from the fast clone" ); diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index b14de19aa9..d4728dd188 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -12843,8 +12843,15 @@ fn typed_f64_receiver_method_clone_raw_loads_after_composed_guards() { // about which emission shape carried it. let inline_probe = caller_ir.find("@PERRY_CLASS_PROTOTYPE_FAST_GUARDS_INVALIDATED"); let method_proof = inline_probe.map_or(method_guard, |p| p.min(method_guard)); + // One-exit class-field GET: at a field-GET site the runtime guard is no + // longer CALLED -- it is the body of `js_class_field_get_ic`, the inline + // pre-check's single miss edge. Other lowerings (the method-override + // receiver-clone gate) still call the guard directly, so accept either + // witness; what this assertion is about is that a field proof exists and + // dominates the typed call, not which symbol carries it. let field_guard = caller_ir - .find("call i32 @js_typed_feedback_class_field_get_guard") + .find("call double @js_class_field_get_ic") + .or_else(|| caller_ir.find("call i32 @js_typed_feedback_class_field_get_guard")) .unwrap_or_else(|| panic!("caller should guard raw-f64 receiver fields:\n{caller_ir}")); // Same for the raw-f64 FIELD proof: one inline class-field precheck // (the field-GET sites' form, first mentioned by its `deref` block in the diff --git a/crates/perry-codegen/tests/typed_feedback.rs b/crates/perry-codegen/tests/typed_feedback.rs index 3ee8f733eb..6dfb6d049e 100644 --- a/crates/perry-codegen/tests/typed_feedback.rs +++ b/crates/perry-codegen/tests/typed_feedback.rs @@ -646,20 +646,28 @@ fn typed_feedback_guards_direct_class_field_specialization() { )); assert!(ir.contains("class_field_set_guard")); - assert!(ir.contains("class_field_get_guard")); assert!(ir.contains("@perry_typed_shape_raw_f64_mask_")); assert!(ir.contains("js_typed_feedback_class_field_set_guard")); - assert!(ir.contains("js_typed_feedback_class_field_get_guard")); assert!(ir.contains("class_field_set.fast")); assert!(ir.contains("class_field_set.fallback")); // #8033: `receiver_class_name` now consults `stable_local_type_proof` // (runtime evidence), which a bare parameter lacks. The numeric-specific // `class_field_get_number.*` path is therefore not selected; the generic - // `class_field_get.*` diamond is emitted instead, with its own fast/fallback - // arms and the typed-feedback guard. The guard assertions above (lines - // 493-499) validate the test's core purpose. + // `class_field_get.*` tower is emitted instead. + // + // One-exit class-field GET: the tower keeps the #5093 inline pre-check and + // the fast slot load, and everything behind the pre-check — the guard call, + // the guard-PASS load, the nullish TypeError and the by-name lookup — is + // now the body of `js_class_field_get_ic`. The guard therefore still runs + // on every miss, one frame deeper; asserting `js_typed_feedback_class_ + // field_get_guard` as a bare substring here would prove nothing either way, + // because the `declare` line alone satisfies it (the same trap #7480's + // comment below describes). Assert the call FORMS instead. assert!(ir.contains("class_field_get.fast")); - assert!(ir.contains("class_field_get.fallback")); + assert!(ir.contains("class_field_inline.deref")); + assert!(ir.contains("call double @js_class_field_get_ic(")); + assert!(!ir.contains("call i32 @js_typed_feedback_class_field_get_guard(")); + assert!(!ir.contains("class_field_get.fallback")); assert!(ir.contains("store double")); assert!(!ir.contains("call void @js_gc_note_slot_layout")); // #5334 lever A: the SET fallback arm collapses to one outlined call; the @@ -670,9 +678,13 @@ fn typed_feedback_guards_direct_class_field_specialization() { // (from the class-field-GET fallback block, not the SET site — the SET copy // was folded into js_class_field_set_fallback by #5334). It is now emitted // only in a typed-feedback build, like the registration call beside it. - // The fallback ARM itself is unchanged and is asserted above/below. + // The fallback ARM itself is asserted above. assert!(!ir.contains("call void @js_typed_feedback_record_fallback_call")); - assert!(ir.contains("call double @js_object_get_field_by_name_f64")); + // The GET miss arm's by-name lookup moved inside `js_class_field_get_ic` + // (which records the fallback call itself, under the same runtime + // `typed_feedback_enabled()` gate the elided emission above respects), so + // the site no longer emits it. + assert!(!ir.contains("call double @js_object_get_field_by_name_f64(")); } /// Body of the first rendered block whose label starts with `label_prefix`, @@ -774,23 +786,27 @@ fn full_outline_ic_collapses_class_field_get_to_single_call() { let _lock = env_lock(); - // Forced ON: one outlined call, no inline get diamond. + // Forced ON: one outlined call and NO tower at all -- not even the #5093 + // inline pre-check, which is the whole point on an oversized module. { let _g = EnvVarGuard::set("PERRY_FULL_OUTLINE_IC", Some("1")); let ir = ir_for(build()); assert!(ir.contains("call double @js_class_field_get_ic")); assert!(!ir.contains("class_field_get.fast")); - assert!(!ir.contains("class_field_get.fallback")); - assert!(!ir.contains("call i32 @js_typed_feedback_class_field_get_guard")); + assert!(!ir.contains("class_field_inline.deref")); + assert!(!ir.contains("call i32 @js_typed_feedback_class_field_get_guard(")); } - // Forced OFF: the inline diamond, no full-outline call. + // Forced OFF: the inline tower. Since the one-exit change, BOTH arms call + // `js_class_field_get_ic`, so the discriminator is the TOWER, not the call: + // OFF keeps the pre-check and the fast slot load, ON has neither. { let _g = EnvVarGuard::set("PERRY_FULL_OUTLINE_IC", Some("0")); let ir = ir_for(build()); - assert!(!ir.contains("call double @js_class_field_get_ic")); + assert!(ir.contains("class_field_inline.deref")); assert!(ir.contains("class_field_get.fast")); - assert!(ir.contains("js_typed_feedback_class_field_get_guard")); + assert!(ir.contains("call double @js_class_field_get_ic")); + assert!(!ir.contains("call i32 @js_typed_feedback_class_field_get_guard(")); } } diff --git a/crates/perry-runtime/src/typed_feedback/tests.rs b/crates/perry-runtime/src/typed_feedback/tests.rs index c12ed2c0c5..4975aa0193 100644 --- a/crates/perry-runtime/src/typed_feedback/tests.rs +++ b/crates/perry-runtime/src/typed_feedback/tests.rs @@ -3038,3 +3038,148 @@ fn function_source_array_literal_keeps_the_array_index_fast_path_armed() { "the emitted guard must still admit plain arrays to the fast path" ); } + +// --------------------------------------------------------------------------- +// `js_class_field_get_ic` — the class-field GET tower's ONE exit. +// +// The helper predates these tests: it was written for the #5391 path-2 full +// outline, which only fires on oversized modules, so its four arms had no +// direct coverage. Every monomorphic `this.field` read whose inline pre-check +// misses now routes through it, so each arm is pinned here: +// +// guard PASS, boxed slot -> the same value the slot holds; +// guard PASS, raw-f64 slot -> the same double, with the intact bit live; +// nullish receiver -> TypeError, not `undefined` (#7153); +// guard FAIL -> one recorded fallback + the by-name answer. +// +// The pass cases assert the site counters too: a helper that silently took the +// fallback on every call would return the right value and pass a value-only +// test, which is exactly the "gate whose subject never ran" shape. +// --------------------------------------------------------------------------- + +// `guards` is a private submodule and its items are not re-exported through +// `super::*`, so name the helper explicitly. +use crate::typed_feedback::guards::js_class_field_get_ic; + +#[test] +fn class_field_get_ic_reads_the_boxed_slot_on_a_guard_pass() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(7401, TypedFeedbackSiteKind::PropertyGet, "obj.x"); + + let class_id = 0x7EED_7401; + let (obj, _, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); + let payload = crate::string::js_string_from_bytes(b"boxed".as_ptr(), 5); + let stored = crate::JSValue::string_ptr(payload); + crate::object::js_object_set_field(obj, 0, stored); + + let got = js_class_field_get_ic(7401, receiver, class_id, expected_shape_id, key_x, 0, 0); + assert_eq!( + got.to_bits(), + stored.bits(), + "a guard PASS must answer with the slot's own NaN-box, bit for bit" + ); + + let site = &typed_feedback_snapshot().sites[0]; + assert_eq!(site.guard_passes, 1); + assert_eq!(site.guard_failures, 0); + assert_eq!( + site.fallback_calls, 0, + "the fast arm must not record a fallback; a helper that always took the \ + by-name path would return the same value" + ); +} + +#[test] +fn class_field_get_ic_reads_the_raw_f64_slot_on_a_guard_pass() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(7402, TypedFeedbackSiteKind::PropertyGet, "obj.x"); + + let class_id = 0x7EED_7402; + let (obj, _, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); + crate::object::js_object_set_field(obj, 0, crate::JSValue::number(5.25)); + let raw_mask = [0b1u64]; + crate::gc::js_gc_init_typed_shape_layout( + obj as u64, + 1, + raw_mask.as_ptr(), + raw_mask.len() as u32, + std::ptr::null(), + 0, + ); + + let got = js_class_field_get_ic(7402, receiver, class_id, expected_shape_id, key_x, 0, 1); + assert_eq!( + got, 5.25, + "a `require_raw_f64` PASS must answer with the raw double the codegen \ + fast block would have loaded" + ); + + let site = &typed_feedback_snapshot().sites[0]; + assert_eq!(site.guard_passes, 1); + assert_eq!(site.fallback_calls, 0); +} + +#[test] +fn class_field_get_ic_records_the_fallback_and_answers_by_name_on_a_guard_fail() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(7403, TypedFeedbackSiteKind::PropertyGet, "obj.x"); + + let class_id = 0x7EED_7403; + let (obj, original_keys, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); + crate::object::js_object_set_field(obj, 0, crate::JSValue::from_bits(5.0f64.to_bits())); + + // Add a key: the shape moves, so the cached (class id, shape id) pair the + // site was compiled against no longer describes this receiver. + let key_y = crate::string::js_string_from_bytes(b"y".as_ptr(), 1); + crate::object::js_object_set_field_by_name(obj, key_y, 10.0); + assert_ne!( + unsafe { crate::object::object_keys_array(obj) }, + original_keys, + "the fixture must actually transition the shape, or the FAIL arm never runs" + ); + + let got = js_class_field_get_ic(7403, receiver, class_id, expected_shape_id, key_x, 0, 0); + assert_eq!( + got.to_bits(), + 5.0f64.to_bits(), + "the FAIL arm must answer by name, not from the stale cached slot" + ); + + let site = &typed_feedback_snapshot().sites[0]; + assert_eq!(site.guard_passes, 0); + assert_eq!(site.guard_failures, 1); + assert_eq!( + site.fallback_calls, 1, + "the helper records the fallback itself — codegen no longer emits the \ + `js_typed_feedback_record_fallback_call` beside it" + ); +} + +#[test] +fn class_field_get_ic_throws_a_type_error_on_a_nullish_receiver() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(7404, TypedFeedbackSiteKind::PropertyGet, "obj.x"); + + let class_id = 0x7EED_7404; + let (obj, _, key_x, _) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); + + for nullish in [crate::value::TAG_UNDEFINED, crate::value::TAG_NULL] { + let receiver = f64::from_bits(nullish); + let threw = catch_runtime_throw(|| { + js_class_field_get_ic(7404, receiver, class_id, expected_shape_id, key_x, 0, 0); + }); + assert!( + threw, + "a field read on a nullish receiver must throw TypeError, not \ + answer `undefined` through the by-name lookup (#7153); bits={nullish:#x}" + ); + } +} diff --git a/crates/perry/tests/issue_8774_argument_shape_clones.rs b/crates/perry/tests/issue_8774_argument_shape_clones.rs index 0c172b5f0c..d36b77b8bd 100644 --- a/crates/perry/tests/issue_8774_argument_shape_clones.rs +++ b/crates/perry/tests/issue_8774_argument_shape_clones.rs @@ -285,7 +285,8 @@ fn stable_argument_clones_are_direct_reported_and_moving_gc_safe() { ); assert!( !clone_body.contains("shape_descriptor_by_id") - && !clone_body.contains("js_typed_feedback_class_field_get_guard"), + && !clone_body.contains("js_typed_feedback_class_field_get_guard") + && !clone_body.contains("js_class_field_get_ic"), "{clone} rebuilt a field IC diamond:\n{clone_body}" ); assert!(ir.contains(&format!("call double @{clone}("))); From 6ed589911470d9ab9047ad18e6840091398b299d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 13:31:01 +0200 Subject: [PATCH 02/10] changelog: fragment for #10189 (cherry picked from commit eb4bee5d5da9851dd99c9b0431bedd355eb3d658) --- changelog.d/10189-class-field-get-one-exit.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog.d/10189-class-field-get-one-exit.md diff --git a/changelog.d/10189-class-field-get-one-exit.md b/changelog.d/10189-class-field-get-one-exit.md new file mode 100644 index 0000000000..7eb9ba9a14 --- /dev/null +++ b/changelog.d/10189-class-field-get-one-exit.md @@ -0,0 +1,10 @@ +Collapse the monomorphic class-field GET tower to one runtime exit. The #5093 +inline precheck and the fast slot load are unchanged; the guard call, the +nullish-throw diamond and the by-name fallback arms are replaced by a single +call to the existing `js_class_field_get_ic`, which performs the same guard, +load, fallback and throw. Per site that is 88 → 47 IR instructions, 3 → 1 +runtime call sites and 6 → 1 RS4GC relocation reloads; the fast path no longer +crosses a statepoint, so it sheds the two relocation reloads it used to carry. +On @babel/parser `.text` shrinks 7.1 % and `.perry_gcmap` 2.0 %, with runtime +instructions and RSS unchanged and `interp.ts` 13 % faster; typed-feedback +guard counters are identical per site. From 93443bec48c8adef3fcb8d08d860f745d887f075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 13:54:43 +0200 Subject: [PATCH 03/10] perf(codegen): collapse the generic property-get tower to two exits Per untyped `obj.prop`, `lower_generic_property_get` expanded 33 basic blocks, 191 pre-RS4GC IR instructions and SIX runtime call sites: an SSO arm, an INT32 class-ref arm, a nullish-throw arm, a non-object-receiver arm, an overflow-slot load, a deleted-slot miss, two Array-subclass named-prefix ladders, and the miss+prime. Every call is a statepoint, so `.text` and `.perry_gcmap` both scale with call sites x live GC values -- on @babel/parser that tower was 29% of all emitted IR across 6,487 sites. The hit is worth its bytes; the arms around it are not. What stays inline is what it was: the receiver-tag test, the small-handle test, the packed header kind/descriptor word, the packed-MRU compare, the overflow-bit test, the inline field load with its hole check, the polymorphic ways (#7753), and the `.length`/`.size` short-circuits. Everything else branches to one of two new runtime entries that reproduce the arms in the same ORDER and with the same cache-priming decisions: js_object_get_field_ic_nonptr(obj_bits, key, site_id) the receiver-tag ladder -- SSO, INT32 class ref, nullish TypeError, and the by-name fallback for every other tag. Takes no cache: none of these arms can prime one. js_object_get_field_ic_slow(obj_handle, key, cache_slot, packed) the heap-pointer arms -- the overflow slot (including its fallback to the THREE-argument miss entry, i.e. WITHOUT republishing the packed word, which is what the old helper did), the deleted-slot miss, the Array-subclass named-prefix proof, and the priming `get_field_ic_miss_impl`. Per site: 33 -> 14 tower blocks, 191 -> 106 IR instructions, 6 -> 2 calls. @babel/parser .text -14.4%, .perry_gcmap -11.7%, O0-fallback units unchanged. TWO exits and not one, measured. With the receiver-tag test and the small-handle test failing to the SAME block, SimplifyCFG folds them into one flat predicate -- `cmp; sete; cmp; setae; test; je` where the chain was `cmp; jne; cmp; ja` -- and every property-read HIT pays +4.00 instructions on a 10M-read monomorphic loop. That is #7883's flat-predicate cost arriving from the optimiser instead of from codegen. Distinct callees keep the chain branchy and let the unmasked receiver bits die in the entry block instead of living across the whole hit path. Three further shapes were needed to finish paying for the hit, each measured on the same 10M-read loop (instructions retired, mine vs base): * the field address is a typed `gep double` rather than `shl 3` + `add`. With an explicit shift the slot is the packed word's last use, so InstCombine folds `(packed >> 32) << 3` into `(packed >> 29) & mask` and isel pays a 10-byte `movabs` plus an `and`; as a GEP index there is no shift to fold and the scaled addressing mode survives. +3.00 -> +1.00 per hit. * the spill-buffer read moved into a `#[cold] #[inline(never)]` `overflow_arm`. Inlined, its `overflow_get` call forced the entry to save callee-saved registers, which gave it a frame and stopped the miss handler from being a sibling call. +49 -> +31 -> +20 per megamorphic miss (the first step was the exit split). * the named-prefix conjunction asks `ObjectMeta` first: that is one load from the header line the entry has already touched, against two dependent loads through the site's cache slot, and an object with no metadata record cannot carry the token. +20 -> +17 per miss. Final micro numbers, instructions retired per read vs base: monomorphic hit +1.00 (one `jmp`, because the hit's and the way's hole checks become congruent after their empty successors fold and SimplifyCFG tail-merges them -- fixing it needs `!prof` branch weights the IR builder cannot emit today), 4-shape polymorphic (the inline ways) -1.24, megamorphic miss +17.02. Registries updated with the new symbols: runtime declarations, the `cold` placement hint, the eh_mode throwing-callee assertion, gc_call_effects' allocating-helper list, and POLL_CAPABLE_RUNTIME in scripts/gc_root_dominance_check.py (both directions of its property-GET self-test fixture now run against both entries -- omitting them would silently re-open the #7154 GET hole, since the calls they replaced no longer appear at those sites). `test-files/test_gap_generic_get_one_exit_arms.ts` is the behavioural witness: every arm that moved out of line, checked against node's own output. PERRY_IC_DIAG counters are identical between the two toolchains on a deterministic fixture (41,216 misses over 5 sites, same per-reason split, same 20,216 primes, same fresh/armed/megamorphic distribution), and the gap-suite subset (10 filters, 290 tests) has byte-identical verdicts. (cherry picked from commit a46769da0aa5cebda74dc2203024637e943c67f7) --- .../src/codegen/trusted_box_callback_tests.rs | 4 +- crates/perry-codegen/src/eh_mode.rs | 16 +- .../src/expr/property_get/generic_dispatch.rs | 834 +++++++----------- .../src/expr/property_get/tests.rs | 299 +++++-- crates/perry-codegen/src/gc_call_effects.rs | 5 + crates/perry-codegen/src/module/linkage.rs | 5 +- .../src/runtime_decls/objects.rs | 11 + .../src/stmt/cached_field_index_return.rs | 6 +- .../src/stmt/element_shape_loop_tests.rs | 5 +- .../tests/native_proof_regressions.rs | 6 +- .../native_proof_regressions/pod_manifest.rs | 6 +- crates/perry-codegen/tests/typed_feedback.rs | 14 +- .../perry-runtime/src/object/field_get_set.rs | 6 + .../src/object/field_get_set/ic_miss.rs | 2 +- .../object/field_get_set/ic_miss/ic_slow.rs | 582 ++++++++++++ crates/perry-transform/src/prop_cse.rs | 12 +- docs/src/internals/gc-rooting-invariant.md | 8 +- scripts/gc_root_dominance_check.py | 63 +- scripts/shape_descriptor_census.py | 2 +- .../test_gap_generic_get_one_exit_arms.ts | 131 +++ 20 files changed, 1383 insertions(+), 634 deletions(-) create mode 100644 crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs create mode 100644 test-files/test_gap_generic_get_one_exit_arms.ts diff --git a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs index 03bbccdb48..89d929b797 100644 --- a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs +++ b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs @@ -347,7 +347,9 @@ fn additive_property_callback_gets_a_cold_deopting_private_body() { ); assert!( special.contains("pic.miss.call") - && special.contains("js_object_get_field_ic_miss") + // T1 renamed the property-GET slow path this cold arm reaches + // (`js_object_get_field_ic_miss_packed` -> the tower's object exit). + && special.contains("js_object_get_field_ic_slow") && special.contains("guarded_add.dynamic") && special.contains("versioned_callback.deopt.mark"), "both observable cold arms must poison the loop before fallback:\n{special}" diff --git a/crates/perry-codegen/src/eh_mode.rs b/crates/perry-codegen/src/eh_mode.rs index 0c436443d0..d694ff75a3 100644 --- a/crates/perry-codegen/src/eh_mode.rs +++ b/crates/perry-codegen/src/eh_mode.rs @@ -105,9 +105,19 @@ mod tests { #[test] fn cold_property_miss_is_still_throwing() { - let name = "js_object_get_field_ic_miss_packed"; - assert_eq!(crate::module::helper_decl_attrs(name), " cold"); - assert!(!callee_is_nothrow(name)); + for name in [ + "js_object_get_field_ic_miss_packed", + // T1: the generic-get tower's two slow exits carry the same `cold` + // placement hint — and the same ability to throw, now including + // the nullish-receiver TypeError the emitted arm used to raise + // inline. A `cold` helper that lost its invoke edge would silently + // swallow every `Cannot read properties of undefined`. + "js_object_get_field_ic_slow", + "js_object_get_field_ic_nonptr", + ] { + assert_eq!(crate::module::helper_decl_attrs(name), " cold", "{name}"); + assert!(!callee_is_nothrow(name), "{name}"); + } assert!(!callee_is_nothrow("js_object_get_field_ic_miss")); assert!(!callee_is_nothrow("unknown_runtime_helper")); } diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 28cb608ffe..889e059eaa 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -157,6 +157,32 @@ pub(crate) fn lower_generic_property_get( return Ok(val); } + // # Inline hit, two exits (T1) + // + // What stays inline below is exactly the hit: the receiver-tag test, the + // small-handle test, the packed header kind/descriptor word, the compact + // MRU compare, the overflow-bit test, the raw slot load with its hole + // check, and the bounded polymorphic ways. EVERY other arm this tower used + // to expand — the SSO receiver, the INT32 class ref, the nullish throw, the + // non-object receiver, the overflow load, the deleted-slot miss, the two + // Array-subclass named-prefix ladders, and the miss+prime — is now a branch + // to one of TWO calls that reproduce them in the same order: + // `js_object_get_field_ic_nonptr` for a receiver that is not a heap + // pointer, `js_object_get_field_ic_slow` for one that is. The split is not + // cosmetic — see `pget.recv_other` below for the +4 instructions per HIT + // that a single shared exit cost. + // + // The arms were not cheap to keep: ~37 basic blocks, ~177 pre-RS4GC IR + // instructions and 6-7 call sites per site, each call a statepoint whose + // live GC values are written into `.perry_gcmap`. On @babel/parser the + // tower was 29% of all emitted IR across 6,487 sites. It is also not a + // trade against the fast path: the hit sequence below is instruction for + // instruction what it was, with ONE deliberate difference — the + // overflow-bit test is spelled `== 0` with its successors swapped, so the + // guard-passing edge is the true edge like every other link in the chain + // (see `pic.hit` below) — and the ways still resolve `PIC_WAYS + 1` shapes + // without a call. + // // Issue #70/#73/#128: guard against non-pointer receivers // before the PIC deref. Tag-based check on the unmasked // NaN-box: real heap references have high-16-bits POINTER_TAG @@ -164,8 +190,9 @@ pub(crate) fn lower_generic_property_get( // to 0x7FFD; everything else (undefined/null/bool=0x7FFC, // int32=0x7FFE, bigint=0x7FFA, plain f64 like 0.0 globalThis // or 3.14, corrupt bit-patterns like 0x00FF_0000_0000 read as - // a BufferHeader) falls through to the invalid branch and - // returns undefined safely. + // a BufferHeader) falls through to the slow entry, whose tag + // ladder throws for a nullish receiver (#462) and routes every + // other tag to the by-name helper. // // Previously used a Darwin mimalloc heap-window check // (`> 2 TB && < 128 TB`). On aarch64-linux-android (issue @@ -176,41 +203,9 @@ pub(crate) fn lower_generic_property_get( // Tag check is platform-independent: same two LLVM ops // (`lshr` + `and`) + one `icmp`, branch-predicted taken. let obj_tag = ctx.block().lshr(I64, &obj_bits, "48"); - // SSO receiver fast path (Step 1.5 of SSO migration). - // SHORT_STRING_TAG = 0x7FF9 can't pass the POINTER/STRING - // check (its masked tag is 0x7FF9, not 0x7FFD) and we - // can't widen the mask because the PIC fast path's - // `*(obj_handle + 16)` would read arbitrary memory from - // the SSO data bits. Instead: check SSO explicitly first, - // route to a dedicated block that calls the SSO-aware - // `js_object_get_field_by_name_f64` runtime entry (which - // handles `.length` directly from the NaN-box length - // byte and returns `undefined` for other keys). - // v0.5.747: INT32-tagged class refs (top16 == 0x7FFE) used - // as PropertyGet receivers. Pre-fix these fell through to - // the invalid-recv path (returning undefined) because the - // 0xFFFD-masked tag check (0x7FFE & 0xFFFD = 0x7FFC, not - // 0x7FFD) treated them as non-pointer values. Drizzle's - // `is(value, type)` chain depends on `Cls.kind` reads through - // an Any-typed local. Refs #420 / #618 followup. - // - // Note: this also catches plain int32 numeric values (e.g. - // `(42).property`). The runtime helper's INT32-tag arm at - // js_object_get_field_by_name returns undefined for any - // class_id not registered in CLASS_DYNAMIC_PROPS, matching - // the previous behavior — pure ints have no static fields. let obj_tag_masked = ctx.block().and(I64, &obj_tag, "65533"); // 0xFFFD let is_valid = ctx.block().icmp_eq(I64, &obj_tag_masked, "32765"); // 0x7FFD - let sso_idx = ctx.new_block("pget.recv_sso"); - let pic_idx = ctx.new_block("pget.recv_ok"); - let invalid_idx = ctx.new_block("pget.recv_bad"); - let class_ref_idx = ctx.new_block("pget.recv_class_ref"); - let final_merge_idx = ctx.new_block("pget.recv_merge"); - let sso_label = ctx.block_label(sso_idx); - let pic_label = ctx.block_label(pic_idx); - let invalid_label = ctx.block_label(invalid_idx); - let class_ref_label = ctx.block_label(class_ref_idx); - let final_merge_label = ctx.block_label(final_merge_idx); + // `.length` on a receiver whose static type is not a proven string. // // The three-arm string-length dispatch in `property_get.rs` (SSO length @@ -221,11 +216,8 @@ pub(crate) fn lower_generic_property_get( // type (`rec.tag.length` where `rec` is an object-literal type, a JSON // `any`, an array element) therefore lands in this generic tower instead, // where a heap string can never be served: the PIC requires a - // GC_TYPE_OBJECT receiver by construction (#72), so EVERY such read misses - // to `js_object_get_field_ic_miss` and walks a ladder built for objects — - // closure-magic deref, buffer and typed-array registry probes, then - // `js_object_get_field_by_name`'s own dispatch, which decodes the key with - // `str::from_utf8` again before reaching the string arm. On `pipeline.ts` + // GC_TYPE_OBJECT receiver by construction (#72), so EVERY such read would + // otherwise call out and walk a ladder built for objects. On `pipeline.ts` // that one read was ~9% of total run time. // // Both string tags are disjoint from POINTER_TAG, so serving them here is @@ -233,11 +225,6 @@ pub(crate) fn lower_generic_property_get( // non-configurable, cannot be shadowed by an own property, and is exactly // what the runtime ladder computes. Everything else keeps the tower. let inline_string_length = property == "length"; - let strlen_heap_idx = if inline_string_length { - Some(ctx.new_block("pget.strlen_heap")) - } else { - None - }; // A dynamically typed `receiver.size` can still be served without the // object PIC when the live receiver is a native Map or Set. Both payloads // start with the same `u32 size` field, and their distinct GcHeader kinds @@ -246,52 +233,105 @@ pub(crate) fn lower_generic_property_get( // as `this.ctx.hooks.size` commonly lose their static Set type, while an // erased annotation alone must never authorize a native-layout load. let inline_collection_size = property == "size"; - let collection_size_idx = if inline_collection_size { - Some(ctx.new_block("pget.collection_size")) - } else { - None - }; - // #7883: the POINTER/STRING test goes FIRST, and the two rare tags are - // discriminated in a cold block off its false edge. The three tag classes - // are pairwise disjoint — `is_valid` is `(tag & 0xFFFD) == 0x7FFD`, true - // only for 0x7FFD/0x7FFF, while SSO is 0x7FF9 and an INT32 class ref is - // 0x7FFE — so testing them in any order gives the same routing. The old - // order (SSO, then class-ref, then pointer) put two 16-bit constant - // materialisations, two compares and two branches in front of every real - // object receiver: 13 instructions before the PIC on the path that is - // taken essentially always. Now it is `lshr` + `and` + `cmp` + branch. + + // A compact per-site word holds the exact ShapeId and slot for the last + // cacheable receiver. The lazily allocated full cache retains bounded + // polymorphic ways and the Array-subclass named-prefix proof. Both are + // materialised before the first branch now: the single slow exit takes + // them as arguments, and every failing guard reaches it. + let cache_name = overridden_cache_name(ctx, object, property) + .unwrap_or_else(|| allocate_property_cache(ctx)); + let cache_slot_ref = format!("@{cache_name}"); + // A compact atomic MRU removes the cache-pointer dependency on a hit. + // The full cache stays lazy and serves prefix/overflow/polymorphic misses. + let packed_site = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let packed_name = format!( + "{}_packed_get", + crate::expr::inline_cache_global_name(ctx, packed_site) + ); + ctx.typed_parse_rodata + .push(format!("@{packed_name} = private global i64 0, align 8")); + let packed_ref = format!("@{packed_name}"); + + let pic_idx = ctx.new_block("pget.recv_ok"); + // The object exit. Named `pic.miss.call` because that is what it still is: + // the block that calls out when the inline cache cannot serve the read. It + // now also lands every receiver-validation failure ON THE POINTER PATH, + // which is what collapses five of the six old call sites into it. + let call_idx = ctx.new_block("pic.miss.call"); + // The non-pointer exit. Its own block and its own callee, deliberately: + // when the receiver-tag test and the small-handle test failed to the SAME + // block, SimplifyCFG folded the two guards into one flat predicate + // (`cmp; sete; cmp; setae; test; je` for `cmp; jne; cmp; ja`) and every + // property-read HIT paid +4.00 instructions — measured on a 10M-read + // monomorphic loop, 1,231,521,261 -> 1,271,522,669 retired. That is + // #7883's flat-predicate cost arriving from the optimiser instead of from + // codegen. Distinct callees keep the chain branchy, and as a bonus the + // unmasked `obj_bits` dies here instead of staying live across the hit + // path for a call that might need it. let other_idx = ctx.new_block("pget.recv_other"); + let merge_idx = ctx.new_block("pget.recv_merge"); + let pic_label = ctx.block_label(pic_idx); + let call_label = ctx.block_label(call_idx); let other_label = ctx.block_label(other_idx); - let check_class_ref_idx = ctx.new_block("pget.check_class_ref"); - let check_class_ref_label = ctx.block_label(check_class_ref_idx); + let merge_label = ctx.block_label(merge_idx); + + // Typed-feedback bookkeeping is COMPILE-TIME gated (off by default), so + // the recording landing block only exists when it has something to record. + // Without it every failing guard branches straight to `pic.miss.call`; + // with it, the guard-fail/fallback-call pair is recorded on exactly the + // edges that recorded it before (`pic.miss.cold` and `pic.miss`), and the + // edges that recorded nothing — the non-pointer tags, the overflow slot, + // a way that turned out to hold a hole — still record nothing. + let cold_idx = + crate::expr::typed_feedback_emission_enabled().then(|| ctx.new_block("pic.miss.cold")); + let cold_label = cold_idx + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| call_label.clone()); + ctx.block().cond_br(&is_valid, &pic_label, &other_label); + + // Off the pointer path only ONE arm is still worth inlining: an SSO + // string's `.length`, which is a byte of the NaN-box itself. Every other + // non-pointer receiver — an SSO string with any other key, an INT32 class + // ref, `null`/`undefined`, a plain double — is the non-pointer entry's tag + // ladder, in the same order it evaluated them here. ctx.current_block = other_idx; - let is_sso = ctx.block().icmp_eq(I64, &obj_tag, "32761"); // 0x7FF9 - ctx.block() - .cond_br(&is_sso, &sso_label, &check_class_ref_label); - ctx.current_block = check_class_ref_idx; - let is_int32_class = ctx.block().icmp_eq(I64, &obj_tag, "32766"); // 0x7FFE - ctx.block() - .cond_br(&is_int32_class, &class_ref_label, &invalid_label); + let sso_arm = inline_string_length.then(|| { + let sso_idx = ctx.new_block("pget.recv_sso"); + let sso_label = ctx.block_label(sso_idx); + let nonptr_idx = ctx.new_block("pget.recv_nonptr"); + let nonptr_label = ctx.block_label(nonptr_idx); + let is_sso = ctx.block().icmp_eq(I64, &obj_tag, "32761"); // 0x7FF9 + ctx.block().cond_br(&is_sso, &sso_label, &nonptr_label); - // Class-ref dispatch: route through the runtime helper which - // detects INT32 class-ref bits and consults CLASS_DYNAMIC_PROPS - // for the static field / dynamic IIFE-set property / synthetic - // `constructor` lookup. Pass full obj_bits (NOT obj_handle — - // the runtime needs the unmasked top16 to detect the tag). - ctx.current_block = class_ref_idx; - let key_handle = emit_key_handle(ctx, &key_handle_global); - let class_ref_result = ctx.block().call( + // `.length` of an SSO string is the length byte in bits 40..47 of the + // NaN-box itself — the same extract `js_object_get_field_by_name_f64` + // performs, minus the call and the key decode. + ctx.current_block = sso_idx; + let len_shifted = ctx.block().lshr(I64, &obj_bits, "40"); + let len_byte = ctx.block().and(I64, &len_shifted, "255"); + let sso_val = ctx.block().uitofp(I64, &len_byte, DOUBLE); + let sso_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + ctx.current_block = nonptr_idx; + (sso_val, sso_end_label) + }); + // The non-pointer exit: SSO / INT32 class ref / nullish throw / everything + // else, in that order, behind one call that needs no cache. + let nonptr_key_handle = emit_key_handle(ctx, &key_handle_global); + let val_nonptr = ctx.block().call( DOUBLE, - "js_typed_feedback_object_get_field_by_name_f64", + "js_object_get_field_ic_nonptr", &[ - (I64, &feedback_site_id), (I64, &obj_bits), - (I64, &key_handle), + (I64, &nonptr_key_handle), + (I64, &feedback_site_id), ], ); - let class_ref_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); + let nonptr_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); ctx.current_block = pic_idx; let observed_key = key_handle_observed.clone().unwrap_or_default(); @@ -309,7 +349,8 @@ pub(crate) fn lower_generic_property_get( // typed-feedback observation on purpose: the site keeps recording every // receiver it sees, so a mixed object/string site cannot be mis-profiled // as monomorphic-object by the arm that is no longer traced here. - if let Some(heap_idx) = strlen_heap_idx { + let strlen_heap_idx = inline_string_length.then(|| { + let heap_idx = ctx.new_block("pget.strlen_heap"); let strlen_heap_label = ctx.block_label(heap_idx); let not_string_idx = ctx.new_block("pget.recv_obj"); let not_string_label = ctx.block_label(not_string_idx); @@ -319,24 +360,8 @@ pub(crate) fn lower_generic_property_get( ctx.block() .cond_br(&is_heap_string, &strlen_heap_label, ¬_string_label); ctx.current_block = not_string_idx; - } - - // A compact per-site word holds the exact ShapeId and slot for the last - // cacheable receiver. The lazily allocated full cache retains bounded - // polymorphic ways and the Array-subclass named-prefix proof. - let cache_name = overridden_cache_name(ctx, object, property) - .unwrap_or_else(|| allocate_property_cache(ctx)); - // A compact atomic MRU removes the cache-pointer dependency on a hit. - // The full cache stays lazy and serves prefix/overflow/polymorphic misses. - let packed_site = ctx.ic_site_counter; - ctx.ic_site_counter += 1; - let packed_name = format!( - "{}_packed_get", - crate::expr::inline_cache_global_name(ctx, packed_site) - ); - ctx.typed_parse_rodata - .push(format!("@{packed_name} = private global i64 0, align 8")); - let packed_ref = format!("@{packed_name}"); + heap_idx + }); // Issue #72: validate the receiver is actually a GC_TYPE_OBJECT // before reading its ShapeId. The receiver @@ -346,20 +371,19 @@ pub(crate) fn lower_generic_property_get( // analysis can't prove `obj.rowsRaw` is an Array — the outer // PropertyGet falls into this generic dispatch) hands the array's // pointer to this PIC. Reading an ObjectHeader ShapeId from that payload - // would be invalid. The slow `js_object_get_field_by_name` - // already routes by `gc_type` (handles Array.length, String.length, - // Set.size, Buffer.length, Error.message, etc.), so funneling - // non-OBJECT receivers through the miss handler fixes correctness - // without giving up the PIC for real objects. + // would be invalid. The slow entry already routes by `gc_type` (it handles + // Array.length, String.length, Set.size, Buffer.length, Error.message, + // etc. through the same miss handler), so funneling non-OBJECT receivers + // to it fixes correctness without giving up the PIC for real objects. // // Issue #340/#341: small-handle guard. Receivers from // native modules (axios, fastify, ioredis, better-sqlite3, // ...) are NaN-boxed POINTER values whose lower-48 is a // small registry id (1, 2, 3, ...). The PIC fast path // below deref's `obj_handle - 8` for the GcHeader byte - // and `obj_handle + 8` for the ShapeId slot — both + // and `obj_handle + 4` for the ShapeId — both // SIGSEGV when `obj_handle` is a small int. Funnel - // small-handle receivers through the slow path so they + // small-handle receivers through the slow entry so they // reach the runtime's `HANDLE_PROPERTY_DISPATCH` table // (axios `r.status` / `r.data`, fastify `req.query` / // `req.params`, etc.). @@ -368,60 +392,43 @@ pub(crate) fn lower_generic_property_get( // detection (raw_ptr < 0x100000). let is_real_ptr = ctx.block().icmp_ugt(I64, &obj_handle, "1048575"); // 0x100000 - // #7883: the hit/miss/merge blocks are minted here so the guard chain - // below can BRANCH OUT to the miss on the first failing predicate - // instead of AND-ing eight of them into one flat `hit`. LLVM if-converts - // a flat predicate, so every receiver paid every load and every compare - // even after the very first one had - // already decided the answer. Each group now ends in its own `cond_br`; - // the miss block reconstructs what the polymorphic-way compares need - // through phis (`false`/`0` on the early-exit edges, which is exactly - // what the flat predicate computed there). - let hit_idx = ctx.new_block("pic.hit"); - let hit_live_idx = ctx.new_block("pic.hit.live"); - let prefix_guard_idx = ctx.new_block("pic.prefix.guard"); - let prefix_meta_idx = ctx.new_block("pic.prefix.meta"); - let prefix_token_idx = ctx.new_block("pic.prefix.token"); - let prefix_hit_idx = ctx.new_block("pic.prefix.hit"); - let desc_classify_idx = ctx.new_block("pic.desc.classify"); - let desc_prefix_guard_idx = ctx.new_block("pic.desc.prefix.guard"); - let desc_prefix_meta_idx = ctx.new_block("pic.desc.prefix.meta"); - let desc_prefix_token_idx = ctx.new_block("pic.desc.prefix.token"); - let desc_prefix_hit_idx = ctx.new_block("pic.desc.prefix.hit"); - let miss_idx = ctx.new_block("pic.miss"); - // #7907: the two receiver-validation failures get their own landing block - // so `pic.miss` is dominated by `pic.token`. See the comment on - // `pic.miss.cold` below for why that is the whole point of this split. - let cold_idx = ctx.new_block("pic.miss.cold"); - let call_idx = ctx.new_block("pic.miss.call"); - let merge_idx = ctx.new_block("pic.merge"); - let hit_label = ctx.block_label(hit_idx); - let hit_live_label = ctx.block_label(hit_live_idx); - let prefix_guard_label = ctx.block_label(prefix_guard_idx); - let prefix_meta_label = ctx.block_label(prefix_meta_idx); - let prefix_token_label = ctx.block_label(prefix_token_idx); - let prefix_hit_label = ctx.block_label(prefix_hit_idx); - let desc_classify_label = ctx.block_label(desc_classify_idx); - let desc_prefix_guard_label = ctx.block_label(desc_prefix_guard_idx); - let desc_prefix_meta_label = ctx.block_label(desc_prefix_meta_idx); - let desc_prefix_token_label = ctx.block_label(desc_prefix_token_idx); - let desc_prefix_hit_label = ctx.block_label(desc_prefix_hit_idx); - let miss_label = ctx.block_label(miss_idx); - let cold_label = ctx.block_label(cold_idx); - let call_label = ctx.block_label(call_idx); - let merge_label = ctx.block_label(merge_idx); + // #7883: the guard chain BRANCHES OUT to the slow exit on the first + // failing predicate instead of AND-ing eight of them into one flat `hit`. + // LLVM if-converts a flat predicate, so every receiver paid every load and + // every compare even after the very first one had already decided the + // answer. let hdr_idx = ctx.new_block("pic.recv_hdr"); let hdr_label = ctx.block_label(hdr_idx); let tok_idx = ctx.new_block("pic.token"); let tok_label = ctx.block_label(tok_idx); + let hit_idx = ctx.new_block("pic.hit"); + // The inline hit's LIVE edge goes straight to the merge unless typed + // feedback has a guard-pass record to put on it. An empty `pic.hit.live` + // is congruent with `pic.way.live`, so SimplifyCFG tail-merges the two and + // the hit path pays a `jmp` to the survivor instead of falling through. + let hit_live_idx = + crate::expr::typed_feedback_emission_enabled().then(|| ctx.new_block("pic.hit.live")); + // The hole edge gets its own landing block, as it did before T1. Note what + // that does and does not buy, measured: `pic.hit.inline` and `pic.way.load` + // end in the same three instructions (bitcast, TAG_HOLE compare, branch), + // and SimplifyCFG folds this block and `pic.way.live` away, so the two + // tails end up congruent and get merged anyway — the hit still reaches the + // shared tail by a `jmp`, and removing this block changed the 10M-read + // monomorphic loop by exactly 0 instructions. It is kept because it keeps + // the emitted hole edge structurally distinct from the way path's, which is + // the shape every reader of this tower since #9287 expects; the remaining + // jump needs branch weights (`!prof`) to fix, which the IR builder has no + // way to emit today. + let deleted_idx = ctx.new_block("pic.hit.deleted"); + let miss_idx = ctx.new_block("pic.miss"); + let hit_label = ctx.block_label(hit_idx); + let deleted_label = ctx.block_label(deleted_idx); + let miss_label = ctx.block_label(miss_idx); // Small-handle receivers (native-module registry ids) must never be // dereferenced. Pre-#7883 they were kept out of the loads by selecting a // sentinel address and AND-ing `is_real_ptr` into `hit`; the branch does // the same job without putting a `select` (and the sentinel's address // materialisation) in front of every real object read. - // A small-handle receiver can never resolve a way (`way_hit` requires a - // real object), so it leaves for `pic.miss.cold` and never enters the - // block the ways live in. ctx.block().cond_br(&is_real_ptr, &hdr_label, &cold_label); ctx.current_block = hdr_idx; @@ -440,33 +447,42 @@ pub(crate) fn lower_generic_property_get( "aarch64" | "arm64" | "arm64_32" | "x86_64" | "i686" | "i386" | "riscv64" | "wasm32" ) .then(|| ctx.block().load(I32, &gc_type_ptr)); - let gc_type = match &packed_header { - Some(word) => ctx.block().trunc(I32, word, I8), - None => ctx.block().load(I8, &gc_type_ptr), - }; + // The separate kind BYTE is needed only by the native-endian descriptor + // guard and by the Map/Set split. With the packed header word the kind is + // already inside it, so materialising the byte anyway would leave a dead + // `trunc` on every hit. + let gc_type = + (inline_collection_size || packed_header.is_none()).then(|| match &packed_header { + Some(word) => ctx.block().trunc(I32, word, I8), + None => ctx.block().load(I8, &gc_type_ptr), + }); // `MapHeader` and `SetHeader` both begin with `size: u32`. A native // collection is not an ObjectHeader and can never hit this PIC, so split // it off immediately after the already-required GC-kind load. The generic // miss handler recognizes the same two kinds before ordinary object // lookup; this only removes that repeated classification and call ladder. - if let Some(collection_idx) = collection_size_idx { + let collection_size_idx = inline_collection_size.then(|| { + let collection_idx = ctx.new_block("pget.collection_size"); let collection_label = ctx.block_label(collection_idx); let object_check_idx = ctx.new_block("pic.recv_object_check"); let object_check_label = ctx.block_label(object_check_idx); - let is_map = ctx.block().icmp_eq(I8, &gc_type, "8"); // GC_TYPE_MAP - let is_set = ctx.block().icmp_eq(I8, &gc_type, "12"); // GC_TYPE_SET + let kind = gc_type + .clone() + .expect("the GC-kind byte is materialised whenever `.size` is inlined"); + let is_map = ctx.block().icmp_eq(I8, &kind, "8"); // GC_TYPE_MAP + let is_set = ctx.block().icmp_eq(I8, &kind, "12"); // GC_TYPE_SET let is_collection = ctx.block().or(I1, &is_map, &is_set); ctx.block() .cond_br(&is_collection, &collection_label, &object_check_label); ctx.current_block = object_check_idx; - } - let is_object_kind = ctx.block().icmp_eq(I8, &gc_type, "2"); + collection_idx + }); // Closures and RegExp values have distinct GC kinds. Every // `GC_TYPE_OBJECT` payload is therefore an ObjectHeader and its ShapeId is // the remaining exact layout discriminator. - + // // #6080: a receiver that has ever had a property/accessor descriptor // installed (`Object.defineProperty`) needs descriptor-aware dispatch — // an accessor must fire on reads, a non-writable slot must reject stores. @@ -476,10 +492,18 @@ pub(crate) fn lower_generic_property_get( // hit path would return the raw slot and bypass the getter entirely. // OBJ_FLAG_HAS_DESCRIPTORS is bit 11 of reserved (bit 27 of the // little-endian header word). Ignore gc_flags and every other flag. + // A descriptor-bearing receiver leaves for the slow exit, which keeps the + // Array-subclass named-prefix proof that used to be a second inline ladder + // here: it is the one case where an unrelated `length` descriptor must not + // make every declared field permanently generic. let is_plain_kind = if let Some(word) = &packed_header { let kind_and_desc = ctx.block().and(I32, word, "134217983"); // 0x080000ff ctx.block().icmp_eq(I32, &kind_and_desc, "2") } else { + let kind = gc_type + .as_ref() + .expect("the GC-kind byte is materialised on native-endian targets"); + let is_object_kind = ctx.block().icmp_eq(I8, kind, "2"); let reserved_addr = ctx.block().sub(I64, &obj_handle, "6"); let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); let reserved = ctx.block().load(crate::types::I16, &reserved_ptr); @@ -487,33 +511,12 @@ pub(crate) fn lower_generic_property_get( let no_desc = ctx.block().icmp_eq(crate::types::I16, &has_desc, "0"); ctx.block().and(I1, &is_object_kind, &no_desc) }; - // Resolve the full cache only after the compact MRU declines a read. - // Loading it here keeps an unused global load on every successful hit. - // Each cold entry retains its own non-null proof before dereferencing. - let cache_slot_ref = format!("@{cache_name}"); let is_plain_object = ctx.block().and(I1, &is_plain_kind, &packed_present); // Validate kind, descriptor policy, and initialized MRU before reading - // ObjectHeader's ShapeId. Header failures bypass the shape ways; the - // descriptor-aware named-prefix path keeps its own full-cache guard. + // ObjectHeader's ShapeId. ctx.block() - .cond_br(&is_plain_object, &tok_label, &desc_classify_label); - - // A descriptor-bearing GC_TYPE_OBJECT normally goes cold. Array-subclass - // `length` is the important exception: runtime can prove that descriptor - // is unrelated to all class-declared named fields and arm word 2. Keep - // this classification off the ordinary descriptor-free hit path. - ctx.current_block = desc_classify_idx; - // #9708: the descriptor prefix path reads cache word 2, so it needs the - // full-cache non-null proof as the shape-miss path; without a cache it is - // simply a cold miss. - let desc_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); - let desc_object_with_cache = ctx.block().and(I1, &is_object_kind, &desc_cache.present); - ctx.block().cond_br( - &desc_object_with_cache, - &desc_prefix_guard_label, - &cold_label, - ); + .cond_br(&is_plain_object, &tok_label, &cold_label); ctx.current_block = tok_idx; // The receiver token is derived solely from its authoritative ShapeId. @@ -539,11 +542,14 @@ pub(crate) fn lower_generic_property_get( let token_miss_label = ctx.block_label(token_miss_idx); ctx.block() .cond_br(&token_eq, &hit_label, &token_miss_label); + + // Every way load still requires a resolved full cache. A site that has + // never primed has no cache, so there is nothing to compare against and + // the read goes straight out. ctx.current_block = token_miss_idx; - // Every subsequent prefix/way load still requires a resolved full cache. let token_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); ctx.block() - .cond_br(&token_cache.present, &prefix_guard_label, &cold_label); + .cond_br(&token_cache.present, &miss_label, &cold_label); // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact @@ -555,216 +561,86 @@ pub(crate) fn lower_generic_property_get( // #9287: the primed slot word may carry IC_SLOT_OVERFLOW_BIT (1 << 30) — // the field lives past the inline region, in the object's spill buffer, // and the inline `obj + header + slot*8` arithmetic below must not run on - // it. Such hits route through `js_object_get_field_ic_overflow_load`, - // which loads through `overflow_get` and falls back to the full miss - // handler on a tombstoned slot. Sites whose field is inline never see the - // bit, so this branch predicts perfectly for them. The polymorphic WAYS - // never hold an encoded slot (`pic_prime_get` refuses to cascade one), so - // only this MRU path needs the check. - let ovf_idx = ctx.new_block("pic.hit.overflow"); + // it. Such hits leave for the slow entry, which re-derives the same MRU + // pair and performs `js_object_get_field_ic_overflow_load`'s `overflow_get` + // (falling back to the full miss handler on a tombstoned slot, without the + // packed republication that helper also did not do). Sites whose field is + // inline never see the bit, so this branch predicts perfectly for them. + // The polymorphic WAYS never hold an encoded slot (`pic_prime_get` refuses + // to cascade one), so only this MRU path needs the check. + // + // Tested as `== 0` rather than `!= 0` so the guard-PASSING edge is the TRUE + // edge, exactly like every other link in this chain. That is not cosmetic: + // `generic_property_get_slot_load_is_reached_only_through_every_guard` + // walks the CFG backwards from the slot load and requires every edge on the + // way to be a true edge, which is what makes a swapped `cond_br` — running + // the raw load when a guard FAILS — turn it red. Before T1 the walk reached + // the load through `pic.hit.overflow` (whose key-handle load also matched + // "load double") and never evaluated this branch's polarity at all. let inline_hit_idx = ctx.new_block("pic.hit.inline"); - let ovf_label = ctx.block_label(ovf_idx); let inline_hit_label = ctx.block_label(inline_hit_idx); let ovf_bits = ctx.block().and(I64, &slot, "1073741824"); // 1 << 30 - let is_ovf = ctx.block().icmp_ne(I64, &ovf_bits, "0"); - ctx.block().cond_br(&is_ovf, &ovf_label, &inline_hit_label); - - ctx.current_block = ovf_idx; - let ovf_key_handle = emit_key_handle(ctx, &key_handle_global); - let ovf_slot_i32 = ctx.block().trunc(I64, &slot, I32); - let val_ovf = ctx.block().call( - DOUBLE, - "js_object_get_field_ic_overflow_load", - &[ - (I64, &obj_handle), - (I64, &ovf_key_handle), - (I32, &ovf_slot_i32), - (PTR, &cache_slot_ref), - ], - ); - let ovf_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); + let is_inline_slot = ctx.block().icmp_eq(I64, &ovf_bits, "0"); + ctx.block() + .cond_br(&is_inline_slot, &inline_hit_label, &call_label); ctx.current_block = inline_hit_idx; - let offset = ctx.block().shl(I64, &slot, "3"); // arm64_32 watchOS: the object fields region begins at // `size_of::()` past the user pointer — 16 on LP64 and // padded ILP32 since #8047. Derive it from the target triple. let obj_header_size = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string(); let base = ctx.block().add(I64, &obj_handle, &obj_header_size); - let field_addr = ctx.block().add(I64, &base, &offset); - let field_ptr = ctx.block().inttoptr(I64, &field_addr); + let base_ptr = ctx.block().inttoptr(I64, &base); + // A typed GEP rather than `shl 3` + `add`: the slot is the LAST use of the + // packed word here, so with an explicit `shl` InstCombine folds + // `(packed >> 32) << 3` into `(packed >> 29) & 0x5fffffff8` and isel pays a + // 10-byte `movabs` plus an `and` for the mask. As a GEP index there is no + // `shl` to fold, so the scaled addressing mode survives — `shr $32` plus + // `(base,idx,8)`, which is what the pre-T1 tower emitted (it kept the plain + // `shr` only because its overflow block consumed the same value). + let field_ptr = ctx.block().gep(DOUBLE, &base_ptr, &[(I64, &slot)]); let val_hit = ctx.block().load(DOUBLE, &field_ptr); let val_hit_bits = ctx.block().bitcast_double_to_i64(&val_hit); let hit_deleted = ctx .block() .icmp_eq(I64, &val_hit_bits, crate::nanbox::TAG_HOLE_I64); - let deleted_idx = ctx.new_block("pic.hit.deleted"); - let deleted_label = ctx.block_label(deleted_idx); + // A hole is a field deleted since priming. It took the ordinary miss + // before (via `pic.miss`, whose way compares can never match a token the + // MRU entry still holds — `pic_prime_get` evicts a duplicate before it + // writes one), and it takes the same ordinary miss now, recording the same + // guard-fail/fallback-call pair on the way. + let hit_live_label = hit_live_idx + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| merge_label.clone()); ctx.block() .cond_br(&hit_deleted, &deleted_label, &hit_live_label); - ctx.current_block = deleted_idx; - let deleted_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); - ctx.block() - .cond_br(&deleted_cache.present, &miss_label, &cold_label); - - ctx.current_block = hit_live_idx; - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_pass", - &[(I64, &feedback_site_id)], - ); - let hit_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); - - // An object-backed Array subclass changes exact ShapeId on every numeric - // push/pop because its elements live in ordinary object slots. Its class- - // declared named prefix does not move. Runtime miss handling proves the - // complete registered prefix plus dense numeric suffix once and publishes - // a nonzero token in cache word 2 and ObjectMeta. Generic structural or - // descriptor transitions clear the object token; only the exact learned - // numeric-tail installer preserves it. - // - // Keep the ordinary miss path cheap: test the cache word first. Every - // non-Array-subclass site reads zero and leaves without touching the - // receiver's meta pointer. - ctx.current_block = prefix_guard_idx; - let cached_prefix_ptr = ctx.block().gep( - I64, - &token_cache.cache, - &[(I64, &PIC_NAMED_PREFIX_TOKEN.to_string())], - ); - let cached_prefix = ctx.block().load(I64, &cached_prefix_ptr); - let prefix_armed = ctx.block().icmp_ne(I64, &cached_prefix, "0"); - ctx.block() - .cond_br(&prefix_armed, &prefix_meta_label, &miss_label); - - // ObjectHeader::meta is the final header field: offset 8 on LP64, 12 on - // ILP32. Load it with the target pointer width, then branch before reading - // ObjectMeta so a null metadata pointer remains harmless. - ctx.current_block = prefix_meta_idx; - let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) { - 4 - } else { - 8 - }; - let meta_offset = - crate::target_layout::object_meta_slot_offset_bytes(ctx.target_triple).to_string(); - let meta_addr = ctx.block().add(I64, &obj_handle, &meta_offset); - let meta_slot = ctx.block().inttoptr(I64, &meta_addr); - let meta_load_ty = if meta_ptr_size == 4 { I32 } else { I64 }; - let meta_raw = ctx.block().load(meta_load_ty, &meta_slot); - let meta = if meta_ptr_size == 4 { - ctx.block().zext(I32, &meta_raw, I64) - } else { - meta_raw - }; - let meta_nonnull = ctx.block().icmp_ne(I64, &meta, "0"); - ctx.block() - .cond_br(&meta_nonnull, &prefix_token_label, &miss_label); - - ctx.current_block = prefix_token_idx; - let meta_ptr = ctx.block().inttoptr(I64, &meta); - // repr(C) ObjectMeta word 6. The first six u64 words are prototype, - // descriptor blooms, flags, spill, and private brand. Runtime has an - // offset assertion paired with the IR test below. - let object_prefix_ptr = ctx.block().gep(I64, &meta_ptr, &[(I64, "6")]); - let object_prefix = ctx.block().load(I64, &object_prefix_ptr); - let prefix_match = ctx.block().icmp_eq(I64, &object_prefix, &cached_prefix); - ctx.block() - .cond_br(&prefix_match, &prefix_hit_label, &miss_label); - - ctx.current_block = prefix_hit_idx; - // The exact ShapeId guard did fail, so preserve typed-feedback accounting - // just like a polymorphic-way hit: the site remains structurally - // polymorphic even though no runtime fallback call is needed. - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_fail", - &[(I64, &feedback_site_id)], - ); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_fallback_call", - &[(I64, &feedback_site_id)], - ); - let prefix_slot_ptr = ctx.block().gep(I64, &token_cache.cache, &[(I64, "1")]); - let prefix_slot = ctx.block().load(I64, &prefix_slot_ptr); - let prefix_offset = ctx.block().shl(I64, &prefix_slot, "3"); - let prefix_base = ctx.block().add(I64, &obj_handle, &obj_header_size); - let prefix_field_addr = ctx.block().add(I64, &prefix_base, &prefix_offset); - let prefix_field_ptr = ctx.block().inttoptr(I64, &prefix_field_addr); - let val_prefix = ctx.block().load(DOUBLE, &prefix_field_ptr); - let prefix_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); - - // Descriptor-bearing Array subclasses reach this duplicate of the family - // guard without ever entering `pic.token`: the exact raw-load PIC remains - // forbidden, but a runtime-proved data-only declared prefix is still safe. - // Every failure goes to the cold handler because the shape token values - // required by `pic.miss` do not dominate this path. - ctx.current_block = desc_prefix_guard_idx; - let desc_cached_prefix_ptr = ctx.block().gep( - I64, - &desc_cache.cache, - &[(I64, &PIC_NAMED_PREFIX_TOKEN.to_string())], - ); - let desc_cached_prefix = ctx.block().load(I64, &desc_cached_prefix_ptr); - let desc_prefix_armed = ctx.block().icmp_ne(I64, &desc_cached_prefix, "0"); - ctx.block() - .cond_br(&desc_prefix_armed, &desc_prefix_meta_label, &cold_label); - - ctx.current_block = desc_prefix_meta_idx; - let desc_meta_addr = ctx.block().add(I64, &obj_handle, &meta_offset); - let desc_meta_slot = ctx.block().inttoptr(I64, &desc_meta_addr); - let desc_meta_raw = ctx.block().load(meta_load_ty, &desc_meta_slot); - let desc_meta = if meta_ptr_size == 4 { - ctx.block().zext(I32, &desc_meta_raw, I64) - } else { - desc_meta_raw + let hit_end_label = match hit_live_idx { + None => ctx.block().label.clone(), + Some(idx) => { + ctx.current_block = idx; + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_guard_pass", + &[(I64, &feedback_site_id)], + ); + let label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + label + } }; - let desc_meta_nonnull = ctx.block().icmp_ne(I64, &desc_meta, "0"); - ctx.block() - .cond_br(&desc_meta_nonnull, &desc_prefix_token_label, &cold_label); - ctx.current_block = desc_prefix_token_idx; - let desc_meta_ptr = ctx.block().inttoptr(I64, &desc_meta); - let desc_object_prefix_ptr = ctx.block().gep(I64, &desc_meta_ptr, &[(I64, "6")]); - let desc_object_prefix = ctx.block().load(I64, &desc_object_prefix_ptr); - let desc_prefix_match = ctx - .block() - .icmp_eq(I64, &desc_object_prefix, &desc_cached_prefix); - ctx.block() - .cond_br(&desc_prefix_match, &desc_prefix_hit_label, &cold_label); - - ctx.current_block = desc_prefix_hit_idx; - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_fail", - &[(I64, &feedback_site_id)], - ); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_fallback_call", - &[(I64, &feedback_site_id)], - ); - let desc_prefix_slot_ptr = ctx.block().gep(I64, &desc_cache.cache, &[(I64, "1")]); - let desc_prefix_slot = ctx.block().load(I64, &desc_prefix_slot_ptr); - let desc_prefix_offset = ctx.block().shl(I64, &desc_prefix_slot, "3"); - let desc_prefix_base = ctx.block().add(I64, &obj_handle, &obj_header_size); - let desc_prefix_field_addr = ctx.block().add(I64, &desc_prefix_base, &desc_prefix_offset); - let desc_prefix_field_ptr = ctx.block().inttoptr(I64, &desc_prefix_field_addr); - let val_desc_prefix = ctx.block().load(DOUBLE, &desc_prefix_field_ptr); - let desc_prefix_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); + // The hit's hole lands here rather than on the shared exit — see the + // congruence note where the block is minted. + ctx.current_block = deleted_idx; + ctx.block().br(&cold_label); // PIC miss on the MRU entry — before paying for the call, try the // polymorphic ways (#7753). // - // `js_object_get_field_ic_miss` is not a cheap fallback: it re-derives the - // receiver kind from scratch (proxy band, closure magic, registered-buffer - // and typed-array registries, small-handle dispatch), reads the + // The slow entry is not a cheap fallback: it re-derives the receiver kind + // from scratch (proxy band, closure magic, registered-buffer and + // typed-array registries, small-handle dispatch), reads the // accessors-in-use thread-local, then linear-scans the keys array with a // `js_string_equals` per key. On a site whose receiver alternates between a // handful of shapes — the shape of every discriminated-union dispatch — @@ -781,9 +657,10 @@ pub(crate) fn lower_generic_property_get( // // # Why this block is DOMINATED by `pic.token` (#7907) // - // Its only predecessor is `pic.token` after the MRU token did not match. - // The exact descriptor identity proves cached-slot bounds, so `token` - // is everything the way compares need. + // Its only predecessor is `pic.token.miss`, which is `pic.token`'s. The + // exact descriptor identity proves cached-slot bounds, so `token` is + // everything the way compares need, and the cache pointer arrives on one + // edge rather than through a phi. // // #7883 could not rely on that: it routed the two receiver-validation // failures here as well, which left the values live on only some edges, so @@ -795,24 +672,8 @@ pub(crate) fn lower_generic_property_get( // the duplicate ladder sat on the hot path. Measured on `interp.ts`'s // `evalNode`, the single hottest instruction in the whole program was the // redundant receiver reconstruction inside this block. - // - // Sending the two validation failures to `pic.miss.cold` instead is what - // establishes the dominance. Nothing about the predicate changed: a - // receiver that fails either check also fails `way_hit` (which ANDs - // `is_object` in), so it could never have resolved a way — the compares - // were dead work for it. ctx.current_block = miss_idx; - // All incoming cache values have passed their own presence guard. The - // merge preserves that proof without resolving the full cache on a hit. - let cache_ref = ctx.block().phi( - PTR, - &[ - (&token_cache.cache, &prefix_guard_label), - (&token_cache.cache, &prefix_meta_label), - (&token_cache.cache, &prefix_token_label), - (&deleted_cache.cache, &deleted_label), - ], - ); + let cache_ref = token_cache.cache.clone(); crate::expr::emit_typed_feedback_record_call( ctx.block(), "js_typed_feedback_record_guard_fail", @@ -915,33 +776,36 @@ pub(crate) fn lower_generic_property_get( let way_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); - // #7907: receiver-validation failure. `way_hit` requires a real pointer to - // a plain descriptor-free `ObjectHeader`, so a receiver that got here can - // never match a way — it goes straight to the handler, which reproduces the - // whole ladder anyway (proxy band, closure magic, buffer/typed-array - // registries, small-handle dispatch). The typed-feedback counters are the - // same two `pic.miss` records on the same edges, so the feedback signal is - // byte-identical to what the merged block reported. - ctx.current_block = cold_idx; - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_fail", - &[(I64, &feedback_site_id)], - ); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_fallback_call", - &[(I64, &feedback_site_id)], - ); - ctx.block().br(&call_label); + // #7907: receiver-validation failure. A receiver that gets here can never + // match a way — the compares require a real pointer to a plain + // descriptor-free `ObjectHeader` — so it goes straight to the handler, + // which reproduces the whole ladder anyway (proxy band, closure magic, + // buffer/typed-array registries, small-handle dispatch). The typed-feedback + // counters are the same two records on the same edges, so the feedback + // signal is byte-identical to what the pre-T1 blocks reported. + if let Some(cold_idx) = cold_idx { + ctx.current_block = cold_idx; + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_guard_fail", + &[(I64, &feedback_site_id)], + ); + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_fallback_call", + &[(I64, &feedback_site_id)], + ); + ctx.block().br(&call_label); + } - // PIC miss: slow path with cache population. + // The object exit: one call reproducing every pointer-path arm this tower + // used to expand. ctx.current_block = call_idx; crate::expr::emit_versioned_loop_callback_deopt(ctx); let miss_key_handle = emit_key_handle(ctx, &key_handle_global); let val_miss = ctx.block().call( DOUBLE, - "js_object_get_field_ic_miss_packed", + "js_object_get_field_ic_slow", &[ (I64, &obj_handle), (I64, &miss_key_handle), @@ -952,149 +816,43 @@ pub(crate) fn lower_generic_property_get( let miss_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); - // Merge PIC hit + way hit + miss, then jump to the outer recv-valid merge. - ctx.current_block = merge_idx; - let pic_val = ctx.block().phi( - DOUBLE, - &[ - (&val_hit, &hit_end_label), - (&val_ovf, &ovf_end_label), - (&val_prefix, &prefix_end_label), - (&val_desc_prefix, &desc_prefix_end_label), - (&val_way, &way_end_label), - (&val_miss, &miss_end_label), - ], - ); - let pic_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); - // Native Map/Set `.size`: their common leading field was admitted only by // the exact live GC-kind checks above. Keep the read inline; calling // `js_map_size` / `js_set_size` would reclassify the same receiver again. - let collection_size_arm = if let Some(collection_idx) = collection_size_idx { + let collection_size_arm = collection_size_idx.map(|collection_idx| { ctx.current_block = collection_idx; let size_i32 = ctx.block().safe_load_i32_from_ptr(&obj_handle); let size = ctx.block().uitofp(I32, &size_i32, DOUBLE); let collection_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); - Some((size, collection_end_label)) - } else { - None - }; - - // Invalid receiver: per JS spec, `undefined` and `null` - // throw a TypeError; other non-pointer tags (int32, bool, - // plain f64, bigint) should auto-box and look up via the - // primitive's prototype. Perry doesn't implement primitive - // auto-boxing yet, so non-nullish primitives continue to - // return `undefined` to preserve existing behavior. - // - // Issue #462: bare `obj.foo` against TAG_UNDEFINED / - // TAG_NULL silently returned undefined, which masked - // unimplemented-API bugs (e.g. `crypto.subtle.encrypt(...)` - // ran to completion as a chain of no-ops). Funnel the - // nullish receiver into the runtime helper which prints a - // node-shaped diagnostic and aborts. - ctx.current_block = invalid_idx; - let is_undef = ctx - .block() - .icmp_eq(I64, &obj_bits, crate::nanbox::TAG_UNDEFINED_I64); - let is_null = ctx - .block() - .icmp_eq(I64, &obj_bits, crate::nanbox::TAG_NULL_I64); - let is_nullish = ctx.block().or(I1, &is_undef, &is_null); - let throw_idx = ctx.new_block("pget.throw_nullish"); - let undef_idx = ctx.new_block("pget.recv_undef_return"); - let throw_label = ctx.block_label(throw_idx); - let undef_label = ctx.block_label(undef_idx); - ctx.block().cond_br(&is_nullish, &throw_label, &undef_label); - - // Throw path: helper aborts the process; block ends with - // `unreachable` because the helper's `-> !` return is - // not visible to LLVM. - ctx.current_block = throw_idx; - let prop_entry = ctx.strings.entry(key_idx); - let prop_bytes_global = format!("@{}", prop_entry.bytes_global); - let prop_len_str = prop_entry.byte_len.to_string(); - let is_null_i32 = ctx.block().zext(I1, &is_null, I32); - ctx.block().call_void( - "js_throw_type_error_property_access", - &[ - (I32, &is_null_i32), - (PTR, &prop_bytes_global), - (I64, &prop_len_str), - ], - ); - ctx.block().unreachable(); - - // Undef-return path: existing fall-through for non-nullish - // invalid receivers. Route through the runtime helper first - // so non-pointer typed shapes can still report a sensible - // value when the runtime knows what they are. Today this - // unblocks Date `.constructor` (Date stores as a raw f64 - // timestamp, so the codegen receiver-tag check at line ~4212 - // rejects it as non-pointer — yet the runtime's - // `js_object_get_field_by_name_f64` recognizes the bit - // pattern via `DATE_REGISTRY` and returns the global Date - // constructor closure). Date-fns `constructFrom` blocker. - ctx.current_block = undef_idx; - let undef_key_handle = emit_key_handle(ctx, &key_handle_global); - let undef_val = ctx.block().call( - DOUBLE, - "js_object_get_field_by_name_f64", - &[(I64, &obj_bits), (I64, &undef_key_handle)], - ); - let invalid_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); - - // SSO receiver: dispatch directly to the runtime by-name - // helper, which reads `.length` inline from the NaN-box - // payload and returns `undefined` for other keys. Bypasses - // the PIC entirely (PIC would read garbage memory). The - // key handle has already been extracted above. - ctx.current_block = sso_idx; - let sso_val = if inline_string_length { - // `.length` of an SSO string is the length byte in bits 40..47 of the - // NaN-box itself — the same extract `js_object_get_field_by_name_f64` - // performs, minus the call and the key decode. - let len_shifted = ctx.block().lshr(I64, &obj_bits, "40"); - let len_byte = ctx.block().and(I64, &len_shifted, "255"); - ctx.block().uitofp(I64, &len_byte, DOUBLE) - } else { - let sso_key_handle = emit_key_handle(ctx, &key_handle_global); - ctx.block().call( - DOUBLE, - "js_object_get_field_by_name_f64", - &[(I64, &obj_bits), (I64, &sso_key_handle)], - ) - }; - let sso_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); + ctx.block().br(&merge_label); + (size, collection_end_label) + }); // Heap string `.length`: `utf16_len` is the leading `u32` of // `StringHeader` — the identical load the proven-string lowering in // `property_get.rs` emits (`strlen.heap`). `safe_load_i32_from_ptr` // keeps a sub-page handle off the load. - let strlen_heap_arm = if let Some(heap_idx) = strlen_heap_idx { + let strlen_heap_arm = strlen_heap_idx.map(|heap_idx| { ctx.current_block = heap_idx; let len_i32 = ctx.block().safe_load_i32_from_ptr(&obj_handle); let heap_len = ctx.block().uitofp(I32, &len_i32, DOUBLE); let heap_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); - Some((heap_len, heap_end_label)) - } else { - None - }; + ctx.block().br(&merge_label); + (heap_len, heap_end_label) + }); - // Outer merge joins PIC result + invalid-receiver undefined - // + SSO result + class-ref dispatch result (+ heap-string `.length`). - ctx.current_block = final_merge_idx; + // One merge for the whole tower: the inline hit, a way hit, the two slow + // calls, and whichever short-circuit arms this key grew. + ctx.current_block = merge_idx; let mut incoming: Vec<(&str, &str)> = vec![ - (&pic_val, &pic_end_label), - (&undef_val, &invalid_end_label), - (&sso_val, &sso_end_label), - (&class_ref_result, &class_ref_end_label), + (&val_hit, &hit_end_label), + (&val_way, &way_end_label), + (&val_miss, &miss_end_label), + (&val_nonptr, &nonptr_end_label), ]; + if let Some((sso_val, sso_end_label)) = sso_arm.as_ref() { + incoming.push((sso_val, sso_end_label)); + } if let Some((heap_len, heap_end_label)) = strlen_heap_arm.as_ref() { incoming.push((heap_len, heap_end_label)); } diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 59f23e97e4..e9a3b0f654 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -136,10 +136,15 @@ fn imported_variable_read_preserves_class_tags_and_calls_the_live_getter_once() (operand == format!("{value} to i64")).then_some(result) }) .expect("getter result must be classified by its intact value tag"); + // #9366's invariant, one indirection later: T1 moved the INT32 class-ref + // arm (and its `js_typed_feedback_object_get_field_by_name_f64` call) into + // `js_object_get_field_ic_nonptr`, which routes on the tag — so the bits it + // receives must still be the getter's UNMASKED value. A masked handle here + // would lose the 0x7FFE tag and the class would dispatch as an object. assert!( ir.lines().any(|line| { - line.contains("call double @js_typed_feedback_object_get_field_by_name_f64(") - && line.contains(&format!(", i64 {bits},")) + line.contains("call double @js_object_get_field_ic_nonptr(") + && line.contains(&format!("i64 {bits},")) }), "class dispatch must receive the getter's unmasked value bits:\n{ir}" ); @@ -391,40 +396,56 @@ fn pic_cache_layout_matches_runtime() { ); } -/// Object-backed Array subclasses mint one ShapeId per numeric tail length. -/// A named-field site on such a receiver must try the independently proved -/// class prefix before falling into the bounded shape ways / runtime miss. +/// Object-backed Array subclasses mint one ShapeId per numeric tail length, so +/// a named-field site on such a receiver is served from the independently +/// proved class prefix rather than the exact ShapeId. +/// +/// Renamed from `generic_property_get_emits_array_subclass_named_prefix_guard` +/// (T1): the proof itself is unchanged and still runs on exactly the same two +/// words, but it runs in `js_object_get_field_ic_slow` instead of in FOUR +/// emitted blocks per site (`pic.prefix.*`) plus FOUR more for the +/// descriptor-bearing twin (`pic.desc.prefix.*`). Its behaviour is pinned by +/// `an_armed_named_prefix_serves_the_cached_slot` in +/// `perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs`; what this test +/// keeps is the CODEGEN half of the contract — the emitted site must hand the +/// runtime the two operands that proof reads, and must not have grown its own +/// copy back. #[test] -fn generic_property_get_emits_array_subclass_named_prefix_guard() { - use crate::expr::property_get::generic_dispatch::PIC_NAMED_PREFIX_TOKEN; - +fn array_subclass_named_prefix_proof_is_reached_through_the_one_exit() { let ir = emit(false, None); - let guard = ir - .find("\npic.prefix.guard") - .unwrap_or_else(|| panic!("expected a named-prefix guard block:\n{ir}")); - let token = ir - .find("\npic.prefix.token") - .unwrap_or_else(|| panic!("expected a named-prefix token block:\n{ir}")); - let hit = ir - .find("\npic.prefix.hit") - .unwrap_or_else(|| panic!("expected a named-prefix hit block:\n{ir}")); - let miss = ir - .find("\npic.miss") - .unwrap_or_else(|| panic!("expected the ordinary PIC miss block:\n{ir}")); - assert!( - guard < token && token < hit && hit < miss, - "prefix guard must precede the ordinary miss path:\n{ir}" - ); - - let guard_body = &ir[guard..token]; - assert!( - guard_body.contains(&format!("i64 {PIC_NAMED_PREFIX_TOKEN}\n")), - "the cheap first gate must read cache word 2 before touching ObjectMeta:\n{guard_body}" - ); - let token_body = &ir[token..hit]; + for gone in [ + "pic.prefix.guard", + "pic.prefix.meta", + "pic.prefix.token", + "pic.prefix.hit", + "pic.desc.classify", + "pic.desc.prefix.guard", + "pic.desc.prefix.meta", + "pic.desc.prefix.token", + "pic.desc.prefix.hit", + ] { + assert!( + !ir.contains(gone), + "the named-prefix ladder must not be emitted per site any more, \ + found `{gone}`:\n{ir}" + ); + } + // The runtime half reads cache word 2 against ObjectMeta word 6 and then + // the cached slot, so the emitted site has to hand it both per-site + // globals — a call that lost either operand would silently stop serving + // Array-subclass named fields and fall back to the full lookup. + let call = ir + .find("\npic.miss.call") + .unwrap_or_else(|| panic!("expected the single slow-exit block:\n{ir}")); + let call_line = ir[call..] + .lines() + .find(|l| l.contains("@js_object_get_field_ic_slow(")) + .unwrap_or_else(|| panic!("expected the one slow call:\n{ir}")); assert!( - token_body.contains("getelementptr i64") && token_body.contains("i64 6\n"), - "ObjectMeta word 6 must carry the runtime-paired prefix token:\n{token_body}" + call_line.contains("ptr @perry_ic_") && call_line.contains("_packed_get"), + "the slow exit must receive BOTH the cache slot (which holds the \ + named-prefix token in word 2) and the packed MRU word, or the runtime \ + cannot reproduce the arms this site stopped emitting:\n{call_line}" ); } @@ -471,8 +492,8 @@ fn generic_property_get_tries_ways_before_calling_the_miss_handler() { miss call — otherwise the compares are not gating anything:\n{ways_body}" ); assert!( - !ways_body.contains("call double @js_object_get_field_ic_miss"), - "the miss call must not sit inside the way block:\n{ways_body}" + !ways_body.contains("call double @js_object_get_field_ic"), + "the slow call must not sit inside the way block:\n{ways_body}" ); // The way compares read (token, slot) pairs at words PIC_WAY_BASE.. and the // gate reads the state word — all inside pic.ways, none anywhere else. @@ -520,10 +541,43 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { main.contains("@perry_ic_"), "test premise: the generic read reaches the inline PIC:\n{ir}" ); + // T1: the landing block is now the single slow exit itself, and the + // dominance is structural — `pic.miss` has exactly ONE predecessor, + // `pic.token.miss`, which `pic.token` dominates. Assert that directly: + // routing any receiver-validation failure back into `pic.miss` would add a + // predecessor and immediately re-introduce the phis #7907 removed. + // `pic.miss` carries a numeric suffix and `pic.miss.call` starts with the + // same text, so match the block's own label exactly and then count the + // branches whose TARGET is that label (a `br i1` naming both blocks counts + // once, for the right one). + let miss_label = main + .lines() + .filter(|l| !l.starts_with(' ') && l.ends_with(':')) + .map(|l| l.trim_end_matches(':')) + .find(|l| { + l.strip_prefix("pic.miss.") + .is_some_and(|tail| tail.chars().all(|c| c.is_ascii_digit())) + }) + .unwrap_or_else(|| panic!("expected a pic.miss block:\n{ir}")) + .to_string(); + let preds = main + .lines() + .filter(|l| l.trim_start().starts_with("br ")) + .filter(|l| { + l.split("label %") + .skip(1) + .any(|t| t.trim_end_matches(&[',', ' '][..]) == miss_label) + }) + .count(); + assert_eq!( + preds, 1, + "pic.miss must have exactly one predecessor (pic.token.miss), or it is \ + no longer dominated by pic.token:\n{ir}" + ); assert!( - main.contains("\npic.miss.cold"), - "the two receiver-validation failures need their own landing block, \ - otherwise pic.miss is not dominated by pic.token:\n{ir}" + main.contains("label %pic.miss.call"), + "every receiver-validation failure must land on the single slow \ + exit:\n{ir}" ); assert!( !main.contains("@PERRY_IC_EPOCH"), @@ -963,28 +1017,56 @@ fn generic_length_read_serves_a_string_inline() { ); // Everything that is NOT a string keeps the tower. assert!( - ir.contains("@perry_ic_") && ir.contains("js_object_get_field_ic_miss"), - "non-string receivers must still reach the inline PIC and its miss \ - handler:\n{ir}" + ir.contains("@perry_ic_") && ir.contains("js_object_get_field_ic_slow"), + "non-string receivers must still reach the inline PIC and its slow \ + exit:\n{ir}" ); } /// The short-circuit is keyed on the property name: any other key on a string -/// receiver (`s.charCodeAt`, `s.constructor`) still needs the runtime, so no -/// other read may grow the string blocks. +/// receiver (`s.charCodeAt`, `s.constructor`) still needs the runtime. +/// +/// T1 moved the SSO arm itself behind the one exit — an SSO receiver with any +/// key but `length` is served by `js_object_get_field_ic_slow`'s tag ladder +/// (`sso_receiver_routes_to_the_by_name_helper` in +/// `ic_miss/ic_slow.rs` pins that it still reaches the by-name helper). What +/// codegen must guarantee is that such a receiver LEAVES: it must never fall +/// into the PIC, whose header loads would read the SSO payload as an address. #[test] fn generic_non_length_read_keeps_the_whole_tower() { let ir = emit_read("charCodeAt"); + for gone in ["pget.strlen_heap", "pget.recv_sso"] { + assert!( + !ir.contains(gone), + "only `.length` may grow an inline string arm, found `{gone}`:\n{ir}" + ); + } + // 32765 = (STRING_TAG|POINTER_TAG) & 0xFFFD: the one test that decides + // whether the receiver may be dereferenced at all. Its false edge must be + // the NON-POINTER exit — a distinct block with a distinct callee, which is + // what stops SimplifyCFG folding this guard into the next one. + let tag_branch = ir + .lines() + .find(|l| l.contains("icmp eq i64") && l.contains("32765")) + .unwrap_or_else(|| panic!("expected the receiver-tag test:\n{ir}")); + let cond = tag_branch + .trim() + .split_once(" = ") + .map(|(lhs, _)| lhs.to_string()) + .unwrap_or_else(|| panic!("malformed tag test: {tag_branch}")); + let branch = ir + .lines() + .find(|l| l.trim_start().starts_with(&format!("br i1 {cond},"))) + .unwrap_or_else(|| panic!("expected a branch on the receiver tag:\n{ir}")); assert!( - !ir.contains("pget.strlen_heap"), - "only `.length` may take the inline string arm:\n{ir}" + branch.contains("label %pget.recv_other") && !branch.contains("label %pic.miss.call"), + "a non-pointer receiver must leave for its OWN exit — sharing the \ + object exit's block is what cost +4 instructions per hit:\n{branch}" ); - let sso = ir - .find("\npget.recv_sso") - .unwrap_or_else(|| panic!("expected an SSO receiver block:\n{ir}")); assert!( - ir[sso..].contains("js_object_get_field_by_name_f64"), - "a non-`length` SSO read must still call the by-name helper:\n{ir}" + ir.contains("@js_object_get_field_ic_slow(") + && ir.contains("@js_object_get_field_ic_nonptr("), + "the tower must still reach both slow entries:\n{ir}" ); } @@ -1015,7 +1097,7 @@ fn generic_size_read_serves_native_collections_inline() { {collection_body}" ); assert!( - ir.contains("@perry_ic_") && ir.contains("js_object_get_field_ic_miss"), + ir.contains("@perry_ic_") && ir.contains("js_object_get_field_ic_slow"), "non-collection receivers must retain the generic property tower:\n{ir}" ); } @@ -1069,10 +1151,18 @@ fn packed_pic_header_guard_is_endianness_aware() { ir.contains("load i16"), "descriptor guard must retain native endianness: {ir}" ); + assert!( + ir.contains(", 2048"), + "the native-endian arm must still test OBJ_FLAG_HAS_DESCRIPTORS: {ir}" + ); } + // T1: the descriptor-bearing fallback is the one exit. It must still + // be a distinct EDGE — a descriptor-bearing receiver may never take + // the raw-slot hit — and the runtime keeps the Array-subclass + // named-prefix exception behind it. assert!( - ir.contains("pic.desc.prefix.guard"), - "descriptor fallback must remain: {ir}" + ir.contains("@js_object_get_field_ic_slow(") && ir.contains("\npic.miss.call"), + "descriptor fallback must remain, through the one exit: {ir}" ); } } @@ -1088,7 +1178,7 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { ir.contains("load atomic i64") && ir.contains("monotonic, align 8"), "{ir}" ); - assert!(ir.contains("@js_object_get_field_ic_miss_packed("), "{ir}"); + assert!(ir.contains("@js_object_get_field_ic_slow("), "{ir}"); assert!( ir.contains("trunc i64") && ir.contains("icmp ne i64"), "{ir}" @@ -1098,3 +1188,104 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { "a full-cache dereference must still guard a null site: {ir}" ); } + +/// T1: the whole point — per untyped `obj.prop` the emitted tower is TWO calls +/// and a handful of blocks, with the inline hit and the polymorphic ways kept. +/// +/// This is a ratchet, so it is an EXACT count in both dimensions. The tower it +/// replaced expanded 33 tower blocks and SIX runtime call sites per site +/// (`js_object_get_field_by_name_f64` twice, the feedback-wrapped class-ref +/// helper, `js_throw_type_error_property_access`, +/// `js_object_get_field_ic_overflow_load`, `js_object_get_field_ic_miss_packed`), +/// each one a statepoint whose live GC values are written into +/// `.perry_gcmap`. On @babel/parser that was 29% of all emitted IR over 6,487 +/// sites; a single arm creeping back inline is a regression measured in +/// megabytes of `.text`, and nothing else in the suite would report it. +/// +/// Two and not one: a single shared exit let SimplifyCFG fold the receiver-tag +/// test and the small-handle test into one flat predicate, costing +4.00 +/// instructions on every HIT (measured). The separate non-pointer callee is +/// what keeps that guard chain branchy, so the count below is 2 — and a change +/// that makes it 1 is a hit-path regression, not a size win. +#[test] +fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { + let ir = emit(false, None); + let func = ir + .split("\ndefine ") + .find(|f| f.contains("\npic.miss.call")) + .unwrap_or_else(|| panic!("no function contains the generic tower:\n{ir}")); + + // Every call/invoke in the whole function, by callee. Feedback records are + // compile-time gated and absent from this build; anything else must be the + // one exit (the fixture's module init contributes its own calls, so match + // on the property-GET family rather than on a total). + let pget_calls: Vec<&str> = func + .lines() + .filter(|l| l.contains(" call ") || l.contains(" invoke ")) + .filter_map(|l| l.split(" @").nth(1)) + .filter_map(|c| c.split('(').next()) + .filter(|c| { + c.starts_with("js_object_get_field") + || c.starts_with("js_typed_feedback_object_get_field") + || *c == "js_throw_type_error_property_access" + }) + .collect(); + let mut sorted = pget_calls.clone(); + sorted.sort(); + assert_eq!( + sorted, + vec![ + "js_object_get_field_ic_nonptr", + "js_object_get_field_ic_slow" + ], + "the tower must expand exactly two property-GET call sites:\n{func}" + ); + + let blocks: Vec<&str> = func + .lines() + .filter(|l| !l.starts_with(' ') && l.ends_with(':')) + .map(|l| l.trim_end_matches(':')) + .filter(|l| l.starts_with("pget.") || l.starts_with("pic.")) + .collect(); + let mut expected = vec![ + // the guard chain and the inline hit + "pget.recv_ok", + // the non-pointer exit, off the tag test's false edge + "pget.recv_other", + "pic.recv_hdr", + "pic.token", + "pic.token.miss", + "pic.hit", + "pic.hit.inline", + // The hit's hole edge keeps its own landing block so its tail is not + // congruent with `pic.way.load`'s; `pic.hit.live` exists only when + // typed feedback has something to record on the live edge. + "pic.hit.deleted", + // the polymorphic ways, deliberately still inline (#7753) + "pic.miss", + "pic.ways", + "pic.way.load", + "pic.way.live", + // the one exit, and the join + "pic.miss.call", + "pget.recv_merge", + ]; + // Labels carry a numeric suffix (`pic.ways.16`); strip it for comparison. + let mut normalized: Vec = blocks + .iter() + .map(|b| { + let mut parts: Vec<&str> = b.split('.').collect(); + if parts.last().is_some_and(|p| p.parse::().is_ok()) { + parts.pop(); + } + parts.join(".") + }) + .collect(); + normalized.sort(); + expected.sort(); + assert_eq!( + normalized, + expected.iter().map(|s| s.to_string()).collect::>(), + "the emitted tower's block set changed:\n{func}" + ); +} diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index e1e8d17fe4..265da914e7 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -750,6 +750,11 @@ mod tests { "js_rel_lt", "js_rel_gt", "js_object_get_field_ic_miss_packed", + // T1: the generic-get tower's two slow exits. One reaches the same + // `get_field_ic_miss_impl`, the other the by-name helper, so both + // allocate and can run a user getter. + "js_object_get_field_ic_slow", + "js_object_get_field_ic_nonptr", ] { assert_ne!( classify_direct_callee(name), diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs index 531516b8ef..823f7525e9 100644 --- a/crates/perry-codegen/src/module/linkage.rs +++ b/crates/perry-codegen/src/module/linkage.rs @@ -251,7 +251,10 @@ pub(crate) fn helper_decl_attrs(name: &str) -> &'static str { // register saves and code layout focused on the inline continuation. // `cold` is only a profitability hint: both calls remain fully // memory-clobbering and the cache miss remains GC-capable/throwing. - "js_object_get_field_ic_miss_packed" | "js_write_barrier_root_nanbox" => " cold", + "js_object_get_field_ic_miss_packed" + | "js_object_get_field_ic_slow" + | "js_object_get_field_ic_nonptr" + | "js_write_barrier_root_nanbox" => " cold", // PURE — each verified: pure bit tests/masking on the f64/i64 args, // total over arbitrary bits, no memory access anywhere in the body. // js_nanbox_pointer value/nanbox.rs — tag ladder, 0 → TAG_NULL diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index e909ba79a7..f35aa1b628 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -366,6 +366,17 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // single call for oversized modules. Args: (obj_bits, key_handle, site_id, // per-site IC cache global) -> field value. module.declare_function("js_object_get_field_ic", DOUBLE, &[I64, I64, I64, PTR]); + // T1: the two exits of the inline generic-get tower. Every guard failure — + // SSO / INT32 class ref / nullish / non-object receiver / overflow slot / + // deleted slot / named prefix / miss+prime — branches to one of these + // instead of expanding its own arm and its own call. + // + // Non-pointer receiver (the emitted tag test failed): (obj_bits, key_handle, + // site_id) -> field value. No cache: none of its arms can prime one. + module.declare_function("js_object_get_field_ic_nonptr", DOUBLE, &[I64, I64, I64]); + // Heap-pointer receiver: (masked obj_handle, key_handle, per-site IC cache + // SLOT, per-site packed MRU word) -> field value. + module.declare_function("js_object_get_field_ic_slow", DOUBLE, &[I64, I64, PTR, PTR]); // Object rest destructuring: copy all properties from src except excluded keys. // Takes a src object ptr and an array of NaN-boxed strings (the excluded keys), // returns a new object pointer. diff --git a/crates/perry-codegen/src/stmt/cached_field_index_return.rs b/crates/perry-codegen/src/stmt/cached_field_index_return.rs index 30339319ab..07759705a5 100644 --- a/crates/perry-codegen/src/stmt/cached_field_index_return.rs +++ b/crates/perry-codegen/src/stmt/cached_field_index_return.rs @@ -207,7 +207,11 @@ pub(super) fn try_emit_cached_field_index_return( .cond_br(&exact, &field_load_label, &prefix_meta_label); ctx.current_block = prefix_meta_idx; - let cached_prefix_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); + // The same cache word the generic property-get tower's named-prefix proof + // uses, and the same one the runtime publishes it in. + let prefix_word = + crate::expr::property_get::generic_dispatch::PIC_NAMED_PREFIX_TOKEN.to_string(); + let cached_prefix_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, &prefix_word)]); let cached_prefix = ctx.block().load(I64, &cached_prefix_ptr); let prefix_armed = ctx.block().icmp_ne(I64, &cached_prefix, "0"); let pointer_bytes = if crate::target_layout::target_is_ilp32(ctx.target_triple) { diff --git a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs index b331ddd133..08e96be315 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs @@ -999,7 +999,10 @@ fn object_literal_element_resolution_does_not_escape_the_clone() { .expect("the merge block should be DEFINED in the emitted IR")..]; assert!( after.contains("js_object_get_field_by_name_f64") - || after.contains("js_object_get_field_ic_miss"), + || after.contains("js_object_get_field_ic_miss") + // T1: the generic tower's cold arms are behind these two entries now. + || after.contains("js_object_get_field_ic_slow") + || after.contains("js_object_get_field_ic_nonptr"), "the post-loop read must stay on the by-name path; emitted:\n{after}" ); } diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index d4728dd188..69480ae817 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -14873,7 +14873,11 @@ fn annotated_class_method_value_uses_generic_lookup() { // fallback, which is the exact regression #8033 exists to prevent. let generic = ir_function_body(&ir, "__probe$generic("); assert!( - generic.contains("call double @js_object_get_field_ic_miss"), + // T1 renamed the tower's cold exits; this assertion is about the + // unguarded body keeping GENERIC lookup, not about which symbol + // serves it. + generic.contains("call double @js_object_get_field_ic_slow") + || generic.contains("call double @js_object_get_field_ic_miss"), "an annotation-only class receiver must preserve generic property \ lookup in the unguarded body:\n{generic}" ); diff --git a/crates/perry-codegen/tests/native_proof_regressions/pod_manifest.rs b/crates/perry-codegen/tests/native_proof_regressions/pod_manifest.rs index 77222de320..d119a37e2c 100644 --- a/crates/perry-codegen/tests/native_proof_regressions/pod_manifest.rs +++ b/crates/perry-codegen/tests/native_proof_regressions/pod_manifest.rs @@ -652,7 +652,11 @@ fn native_pod_view_length_survives_immutable_local_alias() { "an immutable PodView alias must use the validating length helper:\n{ir}" ); assert!( - !ir.contains("call double @js_object_get_field_ic_miss"), + // T1 renamed the tower's exits; a negative assertion that names only + // the retired symbol can no longer fail, so it names the live ones. + !ir.contains("call double @js_object_get_field_ic_slow") + && !ir.contains("call double @js_object_get_field_ic_nonptr") + && !ir.contains("call double @js_object_get_field_ic_miss"), "a PodView alias must not enter the ordinary object-property PIC:\n{ir}" ); } diff --git a/crates/perry-codegen/tests/typed_feedback.rs b/crates/perry-codegen/tests/typed_feedback.rs index 6dfb6d049e..557ad75720 100644 --- a/crates/perry-codegen/tests/typed_feedback.rs +++ b/crates/perry-codegen/tests/typed_feedback.rs @@ -581,8 +581,9 @@ fn a_default_build_emits_no_typed_feedback_recording_calls() { // empty string. The property boundaries themselves must still be here — // this test proves the RECORDING is gone, not the program. assert!( - ir.contains("js_object_get_field_by_name_f64") - || ir.contains("js_object_get_field_ic_miss"), + ir.contains("call double @js_object_get_field_ic_slow(") + || ir.contains("call double @js_object_get_field_by_name_f64(") + || ir.contains("call double @js_object_get_field_ic_miss"), "the property reads themselves must still be lowered; emitted:\n{ir}" ); // And the helpers that DECIDE something, rather than merely counting, are @@ -598,8 +599,15 @@ fn a_default_build_emits_no_typed_feedback_recording_calls() { // the two dispatchers this same fixture still emits -- the property GET // (the set dispatcher's twin) and the method call -- and as CALLS, since // the old symbol match was satisfied by the `declare` line alone. + // T1: the property GET's dispatching wrapper is one indirection further + // out. `js_typed_feedback_object_get_field_by_name_f64` is no longer + // emitted per site — it is the INT32 class-ref arm of + // `js_object_get_field_ic_nonptr`, which the site calls with the same + // `site_id`. The line this assertion draws is unchanged: a dispatcher that + // DECIDES something is emitted in a default build, a helper that merely + // counts is not (all six are asserted absent above). assert!( - ir.contains("call double @js_typed_feedback_object_get_field_by_name_f64("), + ir.contains("call double @js_object_get_field_ic_nonptr("), "dispatching feedback wrappers must still be emitted in a default build \ (property get):\n{ir}" ); diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 318bb493d9..dc9438fcd3 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -309,8 +309,14 @@ pub use ic_miss::{ js_object_set_field_by_property_id, js_private_brand_add, js_private_brand_check, js_private_field_add, js_private_guard, PicCache, PicCacheSlot, PIC_CACHE_WORDS, }; +/// The one slow exit of the emitted generic property-get tower. Declared here +/// rather than inside `ic_miss.rs` only because that file sits at the +/// 2000-line cap; the source lives next to its sibling entries. +#[path = "field_get_set/ic_miss/ic_slow.rs"] +mod ic_slow; pub(crate) use ic_slot::pic_slot_census; pub use ic_slot::{pic_arena_bytes, pic_slot_peek, pic_slot_resolve, pic_slots_resolved}; +pub use ic_slow::{js_object_get_field_ic_nonptr, js_object_get_field_ic_slow}; #[cfg(test)] mod buffer_ic_miss_tests { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index 41a847b423..9cedc25adb 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -537,7 +537,7 @@ fn ic_diag_note( mod packed_get; pub use packed_get::{js_object_get_field_ic_miss, js_object_get_field_ic_miss_packed}; -fn get_field_ic_miss_impl( +pub(super) fn get_field_ic_miss_impl( obj: *const ObjectHeader, key: *const crate::StringHeader, cache_slot: *mut PicCacheSlot, diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs new file mode 100644 index 0000000000..4f36ea44aa --- /dev/null +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -0,0 +1,582 @@ +//! The two slow exits of the emitted generic property-get tower. +//! +//! # Why this exists +//! +//! `lower_generic_property_get` used to emit ~37 basic blocks and 6-7 runtime +//! call sites per untyped `obj.prop`: an SSO arm, an INT32 class-ref arm, a +//! nullish-throw arm, a non-object-receiver arm, an overflow-slot load, a +//! deleted-slot miss, two Array-subclass named-prefix ladders, and the +//! miss+prime. Every one of those calls is a statepoint, so both the emitted +//! `.text` and `.perry_gcmap` scale with call sites x live GC values — on +//! @babel/parser that tower was 29% of all emitted IR over 6,487 sites. +//! +//! The inline *hit* is worth its bytes; the arms around it are not. These two +//! entries are what the site branches to instead of each of them, reproducing +//! the arms they replaced in the same ORDER and with the same cache-priming +//! decisions. The emitted site keeps, byte for byte, the receiver-tag test, the +//! small-handle test, the packed header kind/descriptor test, the packed-MRU +//! compare, the overflow-bit test, the inline field load with its hole check, +//! and the polymorphic ways. +//! +//! # Why TWO exits and not one +//! +//! The first cut had one entry taking the unmasked NaN-box, reached from every +//! guard failure including the receiver-tag test. It cost **+4.00 instructions +//! on every property-read HIT**, measured on a monomorphic 10M-read loop +//! (1,231,521,261 -> 1,271,522,669 retired instructions, +3.2%), and the +//! disassembly says exactly why: with the tag test and the small-handle test +//! failing to the SAME block, LLVM's SimplifyCFG folds them into one flat +//! predicate — +//! +//! ```text +//! cmp …; sete %dl; cmp …; setae %dil; test %dil,%dl; je (6 instructions) +//! ``` +//! +//! — where the branchy chain was `cmp; jne; cmp; ja` (4). That is #7883's +//! finding re-introduced from the other side: there codegen flattened the +//! guards, here the optimiser does it because the shared exit made the two +//! branches congruent. Giving the tag test its own callee restores the chain +//! (and, as a bonus, lets the unmasked `obj_bits` die in the entry block +//! instead of staying live across the whole hit path for a call it might make). +//! +//! The split is along the one line that matters: [`js_object_get_field_ic_nonptr`] +//! serves receivers that are NOT heap pointers (and needs the tag, not the +//! cache), [`js_object_get_field_ic_slow`] serves the ones that are (and needs +//! the cache, not the tag). Two call sites per site instead of six. +//! +//! # The arms, in the order the emitted tower took them +//! +//! 1. Receiver-tag routing ([`js_object_get_field_ic_nonptr`]), exactly as +//! `js_object_get_field_ic` does it (SSO -> the SSO-aware by-name helper; +//! INT32 class ref -> the feedback-wrapped by-name helper; nullish -> +//! `TypeError`; any other non-pointer -> the by-name helper, which still +//! resolves typed shapes such as Date `.constructor`). +//! 2. A packed-MRU token HIT that the emitted hit path declined: +//! * the slot carries `IC_SLOT_OVERFLOW_BIT` — the field lives in the spill +//! buffer, so the inline `obj + header + slot*8` arithmetic must not run +//! on it. This is `js_object_get_field_ic_overflow_load`'s body, including +//! its fallback to the miss handler **without** the packed republication +//! (that helper called the three-argument `js_object_get_field_ic_miss`, +//! and re-publishing the packed word here would change which arm the next +//! read of the same site takes); +//! * otherwise the inline load found `TAG_HOLE` — a field deleted since +//! priming — and takes the ordinary miss, with the packed word, exactly +//! as the emitted `pic.hit.deleted` -> `pic.miss.call` edge did. +//! 3. The Array-subclass named-prefix proof (cache word 2 against `ObjectMeta` +//! word 6, then the cached slot). The emitted tower had this twice, once for +//! a descriptor-free receiver whose exact ShapeId missed and once for a +//! descriptor-bearing one that could never take the raw-load PIC at all; +//! both reduce to the same three loads on a `GC_TYPE_OBJECT` receiver with a +//! resolved cache, so they are one arm here. +//! 4. Everything else: `get_field_ic_miss_impl`, which owns the priming policy +//! (`prime_get`, the megamorphic countdown, the `PERRY_IC_DIAG` rows). +//! +//! # What is deliberately NOT here +//! +//! The polymorphic ways stay inline. They are read-only compares against a +//! cache the site already resolved, they serve the receiver rotation that is +//! the whole point of #7753, and moving them behind a call is a separate +//! measurement. +//! +//! Typed feedback is unchanged: the OBSERVE call still sits inline in +//! `pget.recv_ok` and the guard-pass / guard-fail / fallback-call records still +//! sit on the emitted edges, all under the same compile-time gate. The one +//! feedback call that lives here is the class-ref arm's, which was a +//! feedback-wrapped helper on that edge before and still is. +//! +//! # Rooting +//! +//! Every call these functions make is in TAIL position: the tag-routing arms, +//! the overflow fallback, and `get_field_ic_miss_impl` are all `return`s, and +//! the two probes above them (`overflow_get`, the named-prefix loads) neither +//! allocate nor enter user code. There is therefore no window in which a raw +//! `obj`/`key` argument is named after a collection point, which is the +//! condition a `RuntimeHandleScope` exists to cover (cf. `js_put_value_set_ic_miss`, +//! which re-reads its key to prime AFTER the allocating `js_put_value_set`). +//! Adding a scope here would root nothing and would put a TLS push/pop on every +//! megamorphic read. If an arm ever grows work after a call, it needs the scope +//! at that moment. + +use crate::object::{ObjectHeader, PicCacheSlot}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Cache word 2: the optional Array-subclass class-declared named-prefix +/// token. Mirrors `PIC_NAMED_PREFIX_TOKEN` in +/// `perry-codegen/src/expr/property_get/generic_dispatch.rs`. +const PIC_NAMED_PREFIX_TOKEN: usize = 2; + +/// The exit for a receiver that is NOT a heap pointer: the emitted site's +/// `(tag & 0xFFFD) == 0x7FFD` test just failed. +/// +/// It takes no cache: none of these arms can prime one, which is exactly why +/// splitting them off costs nothing and buys the emitted guard chain back (see +/// the module header). +/// +/// * `obj_bits` — the receiver's full, UNMASKED NaN-box bits. +/// * `key` — the interned property-name `StringHeader`, already masked. +/// * `site_id` — the typed-feedback site id, used only by the class-ref arm. +#[no_mangle] +pub extern "C" fn js_object_get_field_ic_nonptr( + obj_bits: i64, + key: *const crate::StringHeader, + site_id: u64, +) -> f64 { + let bits = obj_bits as u64; + let tag = bits >> 48; + let obj_unmasked = bits as usize as *const ObjectHeader; + + // SSO receiver (SHORT_STRING_TAG): the SSO-aware by-name helper reads + // `.length` from the NaN-box payload and answers undefined otherwise. + // A `.length` site keeps serving this inline and never gets here. + if tag == 0x7FF9 { + return super::js_object_get_field_by_name_f64(obj_unmasked, key); + } + // INT32-tagged class ref: static field / dynamic IIFE-set property / + // synthetic `constructor`. Passes the UNMASKED bits so the runtime can + // detect the tag, exactly as the emitted `pget.recv_class_ref` did. + if tag == 0x7FFE { + return crate::typed_feedback::js_typed_feedback_object_get_field_by_name_f64( + site_id, + obj_unmasked, + key, + ); + } + // `undefined`/`null` throw a node-shaped TypeError (#462). The emitted + // arm passed the property's static bytes and ended in `unreachable`; + // this reads the same bytes off the interned key and the helper is + // `-> !`, so the two are the same divergence with the same message. + if bits == crate::value::TAG_UNDEFINED || bits == crate::value::TAG_NULL { + let is_null = u32::from(bits == crate::value::TAG_NULL); + let (ptr, len) = unsafe { + match super::super::has_own_helpers::str_from_string_header(key) { + Some(s) => (s.as_ptr(), s.len()), + None => (std::ptr::null(), 0), + } + }; + crate::error::js_throw_type_error_property_access(is_null, ptr, len); + } + // Every other tag: no auto-boxing, but the by-name helper still recognizes + // typed shapes (Date `.constructor` via DATE_REGISTRY). A POINTER/STRING + // receiver never reaches this entry from emitted code; if one ever did, + // this arm is the correct — merely non-priming — answer for it too, so + // there is no unverified mode behind the split. + super::js_object_get_field_by_name_f64(obj_unmasked, key) +} + +/// The spill-buffer read, out of line. +/// +/// Kept in its own `#[cold] #[inline(never)]` function for a code-shape reason +/// that is worth 12 instructions on EVERY slow call: inlined, its +/// `overflow_get` call forces `js_object_get_field_ic_slow` to save +/// callee-saved registers across it, which gives that function a real frame +/// (`push rbp/r15/r14/rbx` + the matching pops) and stops the miss handler from +/// being a sibling call. Out of line, the entry's every call is in tail +/// position and it needs no frame at all. +/// +/// Semantics are `js_object_get_field_ic_overflow_load`'s, unchanged: read +/// through `overflow_get`, and on a tombstoned slot fall back to the +/// THREE-argument miss entry — i.e. with a null `packed`, because +/// republishing the packed word here would re-decide which arm the next read +/// of this site takes. +/// +/// # Safety +/// Same contract as the caller: `obj` is a live `GC_TYPE_OBJECT` above the +/// handle band whose shape stamp matched the packed word, and `slot` is that +/// word's high half with `IC_SLOT_OVERFLOW_BIT` set. +#[cold] +#[inline(never)] +unsafe fn overflow_arm( + obj: *const ObjectHeader, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + slot: u32, +) -> f64 { + let idx = (slot & !crate::proxy::IC_SLOT_OVERFLOW_BIT) as usize; + if let Some(v) = crate::object::overflow_get(obj as usize, idx) { + if v != crate::value::TAG_HOLE { + return f64::from_bits(v); + } + } + super::ic_miss::get_field_ic_miss_impl(obj, key, cache_slot, std::ptr::null()) +} + +/// The exit for a receiver that IS a heap pointer — every failing guard on the +/// emitted site's object path lands here. +/// +/// * `obj_handle` — the receiver with the NaN-box tag already masked off. The +/// caller has established the POINTER/STRING tag; this entry re-establishes +/// everything below it, because a guard failure says nothing about WHICH +/// guard failed. +/// * `key` — the interned property-name `StringHeader`, already masked. +/// * `cache_slot` — the site's [`PicCacheSlot`] (`@perry_ic_N`), possibly still +/// null: `pic_slot_peek` answers null and the miss handler resolves it when +/// it actually primes. +/// * `packed` — the site's compact MRU word (`@perry_ic_N_packed_get`). +#[no_mangle] +pub extern "C" fn js_object_get_field_ic_slow( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, +) -> f64 { + let obj = obj_handle as usize as *const ObjectHeader; + let addr = obj as usize; + // Below the handle band the emitted code never dereferenced, and neither + // does this: the miss handler routes registry ids through + // HANDLE_PROPERTY_DISPATCH. + if crate::value::addr_class::is_above_handle_band(addr) { + // SAFETY: the caller's emitted `(tag & 0xFFFD) == 0x7FFD` test + // established a POINTER/STRING NaN-box and the address is above the + // handle band — the same licence under which the emitted tower loaded + // this header word inline. + unsafe { + let header = &*((addr - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader); + if header.obj_type == crate::gc::GC_TYPE_OBJECT { + let plain = header._reserved & crate::gc::OBJ_FLAG_HAS_DESCRIPTORS == 0; + // --- 2. the MRU token hit the emitted hit path declined ----- + if plain && !packed.is_null() { + // The emitted guard is `icmp eq i32 %pcid, trunc(%packed)` + // against the RAW header word, with no range check, and + // this is that test: a nonzero packed word always carries a + // valid ShapeId (`prime_get` publishes nothing else), and a + // receiver whose word is an ordinary class id can never + // equal one, so `object_shape_stamp`'s range check would + // only re-derive what the equality already proves (#809's + // keyless receiver fails the equality, not the range test). + let word = (*packed).load(Ordering::Relaxed); + if word != 0 && (*obj).parent_class_id == word as u32 { + let slot = (word >> 32) as u32; + if slot & crate::proxy::IC_SLOT_OVERFLOW_BIT != 0 { + return overflow_arm(obj, key, cache_slot, slot); + } + // An inline slot that reached this entry on a token hit + // was a `TAG_HOLE` — the field was deleted since + // priming. The emitted `pic.hit.deleted` edge took the + // ordinary miss WITH the packed word. + return super::ic_miss::get_field_ic_miss_impl( + obj, key, cache_slot, packed, + ); + } + } + // --- 3. the Array-subclass named-prefix proof -------------- + // An object-backed Array subclass mints a new exact ShapeId on + // every numeric push/pop, but its class-declared named prefix + // does not move. The runtime proves the complete prefix once + // and publishes one token in cache word 2 and in ObjectMeta; + // any structural or descriptor transition clears the owner's. + // + // The conjunction is ordered by what it costs to ASK, not by + // which half the emitted tower read first: `meta` is one load + // from the header line this function has already touched + // (`obj - 8` and `obj + 4`), while the cache word is two + // dependent loads through the site's slot. An object with no + // metadata record cannot carry the token, so testing it first + // retires the overwhelmingly common non-subclass receiver in + // two instructions instead of eight — worth 6 on every miss, + // and the same conjunction either way. + let meta = (*obj).meta; + if !meta.is_null() { + let cache = crate::object::pic_slot_peek(cache_slot); + if !cache.is_null() + && (*cache)[PIC_NAMED_PREFIX_TOKEN] != 0 + && (*meta).array_subclass_named_prefix_token + == (*cache)[PIC_NAMED_PREFIX_TOKEN] as u64 + { + // Word 1 is an INLINE slot whenever word 2 is armed: an + // overflow prime writes word 2 = 0 precisely so this + // address arithmetic can never see one. + let slot = (*cache)[1] as usize; + let field = (obj as *const u8) + .add(std::mem::size_of::() + slot * 8) + as *const f64; + return *field; + } + } + } + } + } + + // --- 4. everything else: the miss handler owns the priming policy ------- + super::ic_miss::get_field_ic_miss_impl(obj, key, cache_slot, packed) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::object::{PicCache, PIC_CACHE_WORDS}; + + fn key_of(bytes: &[u8]) -> *const crate::StringHeader { + crate::string::js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32) + } + + /// What the emitted site passes: the NaN box with its tag masked off. The + /// tag itself was already tested inline, which is why this entry never + /// sees it. + fn handle(obj: *mut ObjectHeader) -> i64 { + (obj as u64 & 0x0000_FFFF_FFFF_FFFF) as i64 + } + + /// A plain own data read that has never primed: the entry must fall all the + /// way through to the miss handler, answer the field, and leave the site + /// primed exactly as the old `js_object_get_field_ic_miss_packed` edge did. + #[test] + fn plain_miss_answers_the_field_and_primes_both_caches() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + let key = scope.root_string_ptr(key_of(b"ic_slow_plain")); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|k| crate::object::js_object_set_field_by_name(o, k, 7.0)) + }); + + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let v = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(v, 7.0); + assert_ne!(cache[0], 0, "the full cache must be primed by the miss"); + assert_eq!(cache[1], 0, "the first own data field lives in slot 0"); + let word = packed.load(Ordering::Relaxed); + assert_ne!(word, 0, "the compact MRU must be published too"); + assert_eq!(word >> 32, 0, "slot 0, with no overflow bit"); + } + + /// An SSO receiver can never be dereferenced as an ObjectHeader. `.length` + /// is served from the NaN-box payload; any other key answers undefined. + #[test] + fn sso_receiver_routes_to_the_by_name_helper() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let sso = crate::value::JSValue::try_short_string(b"hey").unwrap(); + let key = scope.root_string_ptr(key_of(b"length")); + let v = key.with_const_ptr(|k| js_object_get_field_ic_nonptr(sso.bits() as i64, k, 0)); + assert_eq!(v, 3.0, "the SSO length byte"); + let other = scope.root_string_ptr(key_of(b"nope_not_here")); + let v = other.with_const_ptr(|k| js_object_get_field_ic_nonptr(sso.bits() as i64, k, 0)); + assert_eq!(v.to_bits(), crate::value::TAG_UNDEFINED); + } + + /// The overflow arm, both directions. + /// + /// A field past the inline region primes with `IC_SLOT_OVERFLOW_BIT` + /// (#9287), which is the exact state that sends the emitted MRU hit here + /// instead of doing the `obj + header + slot*8` arithmetic — that address + /// is not where the value lives. The positive direction is the spill read; + /// the negative one is the tombstone, where this entry must fall back to + /// the full miss handler rather than answer the hole. + #[test] + fn an_overflow_slot_reads_through_the_spill_and_falls_back_when_tombstoned() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + // INLINE_SLOT_FLOOR is 2, so the fourth key is past the inline region. + let mut keys = Vec::new(); + for (i, name) in [ + b"ic_slow_ovf_a".as_slice(), + b"ic_slow_ovf_b".as_slice(), + b"ic_slow_ovf_c".as_slice(), + b"ic_slow_ovf_d".as_slice(), + ] + .iter() + .enumerate() + { + let k = scope.root_string_ptr(key_of(name)); + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, i as f64)) + }); + keys.push(k); + } + let key = keys.last().expect("four keys"); + + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let read = |slot: &mut PicCacheSlot, packed: &AtomicU64| { + obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, slot, packed)) + }) + }; + assert_eq!(read(&mut slot, &packed), 3.0, "the priming read"); + let word = packed.load(Ordering::Relaxed); + assert_ne!(word, 0, "test premise: the site primed"); + assert_ne!( + (word >> 32) as u32 & crate::proxy::IC_SLOT_OVERFLOW_BIT, + 0, + "test premise: the fourth field must live past the inline region, \ + or this test never reaches the overflow arm (packed word {word:#x})" + ); + // The hit the emitted code now delegates: same packed pair, read + // through `overflow_get`. + assert_eq!(read(&mut slot, &packed), 3.0, "the overflow-arm read"); + + // Tombstone it. The spill slot no longer holds a value, so the arm must + // decline and let the full miss handler answer for the prototype chain. + obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| { + assert_eq!(crate::object::js_object_delete_field(o, k), 1); + }) + }); + assert_eq!( + read(&mut slot, &packed).to_bits(), + crate::value::TAG_UNDEFINED, + "a tombstoned overflow slot must miss, not answer the hole" + ); + } + + /// A non-nullish, non-pointer receiver keeps the old fall-through: no + /// throw, no cache, `undefined` unless the runtime knows the typed shape. + #[test] + fn plain_number_receiver_returns_undefined_without_priming() { + let _lock = crate::gc::global_side_table_test_lock(); + let key = key_of(b"ic_slow_on_a_double"); + let v = js_object_get_field_ic_nonptr(3.5f64.to_bits() as i64, key, 0); + assert_eq!(v.to_bits(), crate::value::TAG_UNDEFINED); + } + + /// The nullish arm must reach `js_throw_type_error_property_access` with + /// the key's bytes. `js_throw` exits the process when `TRY_DEPTH == 0`, so + /// the assertion here is on the pre-throw routing decision: `undefined` and + /// `null` are the only two bit patterns that take it, which is what the + /// emitted `pget.throw_nullish` predicate tested. + #[test] + fn only_undefined_and_null_take_the_throwing_arm() { + for bits in [crate::value::TAG_UNDEFINED, crate::value::TAG_NULL] { + assert!( + (bits >> 48) & 0xFFFD != 0x7FFD, + "a nullish tag must not look like a heap pointer" + ); + } + // TAG_FALSE/TAG_TRUE sit in the same 0x7FFC band and must NOT throw. + let key = key_of(b"ic_slow_on_a_bool"); + let v = js_object_get_field_ic_nonptr(crate::value::TAG_TRUE as i64, key, 0); + assert_eq!(v.to_bits(), crate::value::TAG_UNDEFINED); + } + + /// A hole in the primed inline slot (the field was deleted) must take the + /// ordinary miss and answer what the full lookup answers — never the raw + /// `TAG_HOLE` word. + #[test] + fn a_deleted_inline_slot_takes_the_ordinary_miss() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + let key = scope.root_string_ptr(key_of(b"ic_slow_deleted")); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|k| crate::object::js_object_set_field_by_name(o, k, 5.0)) + }); + + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let first = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(first, 5.0); + assert_ne!(packed.load(Ordering::Relaxed), 0, "test premise: primed"); + + obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| { + assert_eq!(crate::object::js_object_delete_field(o, k), 1); + }) + }); + let after = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!( + after.to_bits(), + crate::value::TAG_UNDEFINED, + "a deleted field reads undefined, not the hole word" + ); + } + + /// The named-prefix arm: an armed cache word 2 that matches the receiver's + /// ObjectMeta token serves the cached slot WITHOUT consulting the exact + /// ShapeId — that is the whole point of the proof, since an object-backed + /// Array subclass re-shapes on every numeric write. + /// + /// Driven directly against the two words the emitted ladder read, so it + /// pins this entry's contract with codegen rather than the subclass + /// machinery that publishes the token. + #[test] + fn an_armed_named_prefix_serves_the_cached_slot() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + let key = scope.root_string_ptr(key_of(b"ic_slow_prefix")); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|k| crate::object::js_object_set_field_by_name(o, k, 11.0)) + }); + // Give the receiver a meta record and an arbitrary prefix token. + const TOKEN: u64 = 0xA11CE; + let (meta, _) = obj.across_mut::(|| unsafe { + crate::object::object_meta_ensure(obj.get_raw_mut_ptr::()) + }); + assert!( + !meta.is_null(), + "test premise: the receiver has a meta record" + ); + unsafe { (*meta).array_subclass_named_prefix_token = TOKEN }; + + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + cache[PIC_NAMED_PREFIX_TOKEN] = TOKEN as i64; + cache[1] = 0; // the field's inline slot + // Word 0 stays 0 and `packed` stays 0, so the exact-ShapeId + // arm cannot serve this read: only the prefix proof can. + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let v = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(v, 11.0); + assert_eq!( + packed.load(Ordering::Relaxed), + 0, + "a prefix hit answers from the cache and must not prime" + ); + + // A token that no longer matches the receiver's must fall through to + // the ordinary lookup (which still answers, and primes). + cache[PIC_NAMED_PREFIX_TOKEN] = (TOKEN + 1) as i64; + let v = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(v, 11.0); + assert_ne!( + packed.load(Ordering::Relaxed), + 0, + "a stale prefix token must reach the priming miss handler" + ); + } + + /// A site whose cache slot has not been resolved yet reads as a null + /// cache, and the named-prefix arm must not dereference it. + /// + /// The read is of a key the receiver does NOT own, so the miss handler + /// answers from the prototype chain without priming — which is what keeps + /// this test on the arm it is about. (Codegen always passes the address of + /// a real `@perry_ic_N` global; a genuinely null slot only ever reaches a + /// runtime entry from a test or from the write PIC's poly tail.) + #[test] + fn an_unresolved_cache_slot_is_never_dereferenced() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + let own = scope.root_string_ptr(key_of(b"ic_slow_present")); + obj.with_mut_ptr(|o| { + own.with_const_ptr(|k| crate::object::js_object_set_field_by_name(o, k, 3.0)) + }); + let absent = scope.root_string_ptr(key_of(b"ic_slow_absent")); + let packed = AtomicU64::new(0); + // A never-published slot: `pic_slot_peek` must answer null and the + // prefix arm must skip rather than read word 2 out of nothing. + let mut slot: PicCacheSlot = std::ptr::null_mut(); + let v = obj.with_mut_ptr(|o: *mut ObjectHeader| { + absent.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(v.to_bits(), crate::value::TAG_UNDEFINED); + assert!(slot.is_null(), "an absent key must not resolve a cache"); + assert_eq!(packed.load(Ordering::Relaxed), 0, "and must not prime"); + } +} diff --git a/crates/perry-transform/src/prop_cse.rs b/crates/perry-transform/src/prop_cse.rs index 7678c2d65f..3b0089fd9f 100644 --- a/crates/perry-transform/src/prop_cse.rs +++ b/crates/perry-transform/src/prop_cse.rs @@ -15,18 +15,18 @@ //! ``` //! //! Every one of those `n.kind` reads compiles to a **full generic -//! property-get diamond** — `pget.recv_sso` / `pget.check_class_ref` / -//! `pget.recv_ok` / the monomorphic-IC guard ladder / `pic.hit` / `pic.miss` / -//! `pget.recv_merge` — about fifty instructions and six branches per site +//! property-get tower** — `pget.recv_ok` / the monomorphic-IC guard ladder / +//! `pic.hit` / `pic.miss` / the polymorphic ways / `pic.miss.call` / +//! `pget.recv_merge` — about forty instructions and six branches per site //! (`expr/property_get/generic_dispatch.rs`). `gc-handoff/apps/interp.ts`'s //! `evalNode` has **53 such sites**, of which `n.kind` accounts for seven and //! `n.op` for seven more. //! //! LLVM `-O3` cannot remove them. GVN dedupes *instructions*, not congruent //! control-flow regions, and every arm of the diamond that is not the inline -//! fast load ends in an opaque call (`js_object_get_field_ic_miss`, -//! `js_object_get_field_by_name_f64`) whose memory effects license nothing. So -//! the redundancy has to be removed before it becomes a diamond — here. +//! fast load ends in an opaque call (`js_object_get_field_ic_slow`) whose +//! memory effects license nothing. So +//! the redundancy has to be removed before it becomes a tower — here. //! //! # The rewrite //! diff --git a/docs/src/internals/gc-rooting-invariant.md b/docs/src/internals/gc-rooting-invariant.md index 78dbcf7ff9..20aab1cb9f 100644 --- a/docs/src/internals/gc-rooting-invariant.md +++ b/docs/src/internals/gc-rooting-invariant.md @@ -275,6 +275,8 @@ helper called from a loop body is in-loop too. `movers` and `UnrootedAlloca`) counts `js_gc_loop_safepoint`, anything in `poll_reaching`, **and** anything in `POLL_CAPABLE_RUNTIME` — the runtime helpers that can re-enter JS, such as `js_object_set_field_by_name`, +`js_object_get_field_ic_slow` (the generic property-GET tower's single slow +exit, and since T1 the only GET symbol most emitted sites carry), `js_object_get_field_ic_miss` and `js_closure_call1`. Those are moving with no poll anywhere near them. @@ -293,7 +295,11 @@ like coverage while doing it. Two rounds of this have now been measured: SET verbatim but lowers every GET to `js_object_get_field_by_name_f64`, `js_object_get_field_ic_miss` or `js_typed_feedback_object_get_field_by_name_f64`, none of which were in the - set. Property sets classified `MOVING: YES`, property gets classified + set. (T1 later collapsed the generic tower's six cold arms into one + `js_object_get_field_ic_slow` call, which is in the set for the same reason + and is now the GET symbol a stale-register window is most likely to span — + a size optimisation is exactly the kind of change that can silently retire + the symbol an audit is keyed on.) Property sets classified `MOVING: YES`, property gets classified `MOVING: no`, and 31 `--stale-registers` hits on the gate corpus were dropped by `--moving-only` as a result — including the shape that faults deterministically under `PERRY_GC_PROTECT_FROMSPACE=1 diff --git a/scripts/gc_root_dominance_check.py b/scripts/gc_root_dominance_check.py index d84642e652..015a5f9590 100755 --- a/scripts/gc_root_dominance_check.py +++ b/scripts/gc_root_dominance_check.py @@ -1261,6 +1261,16 @@ def is_collecting(callee): # Same getter/Proxy-capable implementation as the retained three-arg ABI; # the fourth argument only adds a numeric packed cache publication. "js_object_get_field_ic_miss_packed", + # T1: the two slow exits of the inline generic-get tower, and now the ONLY + # property-GET symbols most emitted sites carry. `_slow` reaches + # `get_field_ic_miss_impl` (hence `js_object_get_field_by_name`, hence a + # user getter and `js_proxy_get`); `_nonptr` reaches + # `js_object_get_field_by_name_f64` on every arm. A register held across + # either is exactly as stale as one held across the entries they replaced. + # Omitting them would silently re-open the #7154 GET hole that this set + # exists to close, because the calls they replaced no longer appear at + # those sites. + "js_object_get_field_ic_slow", "js_object_get_field_ic_nonptr", "js_typed_feedback_object_get_field_by_name_f64", "js_array_sort_default", "js_array_sort_with_comparator", "js_array_map", "js_array_filter", "js_typed_array_for_each", @@ -5057,6 +5067,8 @@ def self_test(): _pget_family = {"js_object_get_field_by_name_f64", "js_object_get_field_ic_miss", "js_object_get_field_ic_miss_packed", + "js_object_get_field_ic_slow", + "js_object_get_field_ic_nonptr", "js_typed_feedback_object_get_field_by_name_f64"} if not _pget_family <= POLL_CAPABLE_RUNTIME: print("self-test FAIL: the emitted property-GET dispatch " @@ -5165,27 +5177,36 @@ def self_test(): "code shape rather than on the staleness.", file=sys.stderr) ok = False - # The packed entry is the name emitted by generic property reads now. - # Keeping only the old ABI above made --moving-only silently discard - # the same stale-slot hazard even though both call the same getter. - for label, source, want in ( - ("stale", _SELFTEST_PROPERTY_GET_WINDOW, 1), - ("reloaded", _SELFTEST_PROPERTY_GET_RELOADED, 0)): - packed = os.path.join(td, f"property_get_packed_{label}.ll") - source = source.replace( - "@js_object_get_field_ic_miss(", - "@js_object_get_field_ic_miss_packed(").replace( - "ptr @perry_ic_1)", - "ptr @perry_ic_1, ptr @perry_ic_packed_1)") - with open(packed, "w") as fh: - fh.write(source) - for moving_only in (False, True): - found = _stale_kinds_probe(packed, moving_only=moving_only) - if found.get("slotload", 0) != want: - print(f"self-test FAIL: packed GET {label}, " - f"moving_only={moving_only}: expected {want} " - f"slotload hazards, got {found!r}", file=sys.stderr) - ok = False + # The packed entry is the name emitted by generic property reads now, + # and since T1 the SLOW entry is the only one most sites carry at all: + # the tower's six other calls collapsed into it, so a set that named + # only the older ABIs would classify almost every real property-GET + # window `MOVING: no` — the exact #7154 hole, reopened by a size + # optimisation rather than by a misspelling. Both are asserted in both + # directions against the same fixture. + for entry, extra_args in ( + ("js_object_get_field_ic_miss_packed", + "ptr @perry_ic_1, ptr @perry_ic_packed_1)"), + ("js_object_get_field_ic_slow", + "ptr @perry_ic_1, ptr @perry_ic_packed_1)"), + ("js_object_get_field_ic_nonptr", "i64 4242)")): + for label, source, want in ( + ("stale", _SELFTEST_PROPERTY_GET_WINDOW, 1), + ("reloaded", _SELFTEST_PROPERTY_GET_RELOADED, 0)): + packed = os.path.join(td, f"property_get_{entry}_{label}.ll") + source = source.replace( + "@js_object_get_field_ic_miss(", + "@%s(" % entry).replace( + "ptr @perry_ic_1)", extra_args) + with open(packed, "w") as fh: + fh.write(source) + for moving_only in (False, True): + found = _stale_kinds_probe(packed, moving_only=moving_only) + if found.get("slotload", 0) != want: + print(f"self-test FAIL: {entry} GET {label}, " + f"moving_only={moving_only}: expected {want} " + f"slotload hazards, got {found!r}", file=sys.stderr) + ok = False if (not is_collecting("js_string_compare_value") or is_collecting("js_string_compare")): print("self-test FAIL: only the raw primitive string comparison " diff --git a/scripts/shape_descriptor_census.py b/scripts/shape_descriptor_census.py index 0626294170..9815d926e0 100644 --- a/scripts/shape_descriptor_census.py +++ b/scripts/shape_descriptor_census.py @@ -690,7 +690,7 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: for fragment in ( 'letpacked_present=ctx.block().icmp_ne(I64,&packed_word,"0");', 'letis_plain_object=ctx.block().and(I1,&is_plain_kind,&packed_present);', - 'cond_br(&is_plain_object,&tok_label,&desc_classify_label)', + 'cond_br(&is_plain_object,&tok_label,&cold_label)', 'letpacked_stamp=ctx.block().trunc(I64,&packed_word,I32);', 'lettoken_eq=ctx.block().icmp_eq(I32,&pcid,&packed_stamp);', 'cond_br(&token_eq,&hit_label,&token_miss_label)', diff --git a/test-files/test_gap_generic_get_one_exit_arms.ts b/test-files/test_gap_generic_get_one_exit_arms.ts new file mode 100644 index 0000000000..9c831842bd --- /dev/null +++ b/test-files/test_gap_generic_get_one_exit_arms.ts @@ -0,0 +1,131 @@ +// Every arm of the generic property-get tower, against node's own output. +// +// `lower_generic_property_get` used to expand each of these as its own inline +// block group with its own runtime call: the SSO receiver, the heap-string +// receiver, the INT32 class ref, the nullish TypeError, the non-object +// receiver, the native Map/Set `.size`, the overflow slot, the deleted slot, +// and the Array-subclass named-prefix proof. They now live behind two runtime +// entries (`js_object_get_field_ic_nonptr` for a receiver that is not a heap +// pointer, `js_object_get_field_ic_slow` for one that is), so a defect in any +// of them is no longer a defect in emitted code that some other test might +// notice — it is a defect in one shared function, and this is the witness that +// each arm still answers what node answers. +// +// Every read below goes through `read()`, whose parameter is `any`: that is +// what denies the front end a type proof and lands the access in the generic +// tower rather than in one of the specialised lowerings. + +function read(o: any, k: string): any { + // One generic `obj.prop` site per arm. + if (k === "a") return o.a; + if (k === "length") return o.length; + if (k === "size") return o.size; + if (k === "kind") return o.kind; + if (k === "charCodeAt") return o.charCodeAt; + return o.zzz; +} + +// --- receiver tags that are not heap pointers ------------------------------- + +// SSO string (short enough to live inside the NaN box) and a heap string. +const sso: any = "hi"; +const heap: any = "a considerably longer string"; +console.log("sso.length", read(sso, "length")); +console.log("heap.length", read(heap, "length")); +console.log("sso.charCodeAt is fn", typeof read(sso, "charCodeAt")); +console.log("heap.zzz", read(heap, "zzz")); + +// A class object read as a value: its static field, not an instance field. +class C { + static kind = "static-c"; + x = 1; +} +const cref: any = C; +console.log("C.kind", read(cref, "kind")); + +// Non-nullish primitives: no auto-boxing, no throw, just `undefined`. +console.log("3.5.zzz", read(3.5 as any, "zzz")); +console.log("true.zzz", read(true as any, "zzz")); + +// --- heap receivers that are not ordinary objects --------------------------- + +const m: any = new Map(); +m.set(1, 2); +const st: any = new Set(); +st.add(9); +console.log("map.size", read(m, "size"), "set.size", read(st, "size")); + +const arr: any = [1, 2, 3]; +console.log("arr.length", read(arr, "length")); + +// --- ordinary objects: hit, hole, re-add ------------------------------------ + +const o: any = { a: 1, b: 2 }; +console.log("o.a", read(o, "a")); +delete o.a; +console.log("o.a deleted", read(o, "a")); +o.a = 7; +console.log("o.a again", read(o, "a")); + +// A field past the inline region (the overflow slot), read repeatedly so the +// site primes and then takes the primed path, then tombstoned. +const big: any = {}; +big.p0 = 0; +big.p1 = 1; +big.p2 = 2; +big.p3 = 3; +big.p4 = 4; +for (let i = 0; i < 3; i++) console.log("big.p4", big.p4); +delete big.p4; +console.log("big.p4 deleted", big.p4); + +// Absent own key: the prototype chain answers. +const proto = { inherited: "yes" }; +const child: any = Object.create(proto); +console.log("child.inherited", child.inherited); + +// A descriptor-bearing receiver: the accessor must fire on EVERY read, which +// is what forbids a raw-slot hit from ever being primed for it. +let calls = 0; +const acc: any = {}; +Object.defineProperty(acc, "g", { + get() { + calls += 1; + return calls; + }, +}); +console.log("acc.g", acc.g, acc.g, acc.g, "calls", calls); + +// --- nullish receivers throw a node-shaped TypeError ------------------------ + +try { + const n: any = null; + console.log(n.foo); +} catch (e: any) { + console.log("caught null:", e.message); +} +try { + const u: any = undefined; + console.log(u.bar); +} catch (e: any) { + console.log("caught undefined:", e.message); +} + +// --- shape rotation: the bounded polymorphic ways --------------------------- + +class S1 { + x = 1; +} +class S2 { + a = 0; + x = 2; +} +class S3 { + a = 0; + b = 0; + x = 3; +} +const rot: any[] = [new S1(), new S2(), new S3()]; +let sum = 0; +for (let i = 0; i < 300; i++) sum += rot[i % rot.length].x; +console.log("poly sum", sum); From 668ae9e3fe885173e56fb71b7733805e384817f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 14:00:05 +0200 Subject: [PATCH 04/10] changelog: fragment for #10196 (cherry picked from commit fe2a9cfd94c5555ea11df88c2d81063298fa1425) --- changelog.d/10196-generic-get-two-exits.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 changelog.d/10196-generic-get-two-exits.md diff --git a/changelog.d/10196-generic-get-two-exits.md b/changelog.d/10196-generic-get-two-exits.md new file mode 100644 index 0000000000..9ba6cff0dc --- /dev/null +++ b/changelog.d/10196-generic-get-two-exits.md @@ -0,0 +1,12 @@ +Collapse the generic (untyped) property-get inline cache from a 33-block diamond +with six runtime call sites to the inline hit path plus the polymorphic ways and +two out-of-line exits: `js_object_get_field_ic_nonptr` for non-pointer receivers +(SSO, class refs, nullish throw, primitives) and `js_object_get_field_ic_slow` +for a pointer receiver that failed a guard (overflow slot, Array-subclass +named-prefix proofs, miss and prime). Per site: 191 → 106 IR instructions, +6 → 2 call sites. The packed-MRU hit path, the ways and every cache decision are +unchanged; the field address is emitted as a typed `gep` so instruction selection +keeps the scaled addressing mode. On @babel/parser `.text` shrinks 14.4 % and +`.perry_gcmap` 11.7 %; total benchmark instructions drop 0.5 % (`interp.ts` +−1.5 %) with RSS unchanged. A new gap test exercises every arm that moved out of +line against Node. From 845bf723c58d3ac79aa4b3f3babf7c33b670b28f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 12:34:02 +0200 Subject: [PATCH 05/10] perf(codegen): extend the element-shape loop clone to the fields and random access shapes Three additions on top of #10171's shape-keyed arm, all needed together for the JSON access benchmark's two remaining shapes: - a LOOP-CARRIED index (`c = (a*c + b) % m; ... rows[c]`), folded to one affine pair and evaluated as `srem i64`, with the write-back placed at the END of the iteration so a mid-iteration side exit cannot double-apply the recurrence; - K accumulator statements folded into one, so the whole iteration commits once, past every side exit it can take; - `arr[i].prop.length` on a string field and `arr[i].prop ? A : B` on a boolean one, each tag-testing the loaded word and side-exiting otherwise. Plus a shared once-per-iteration element deref/residual check, hung off the body's leading virtual binding, so three reads of one element pay one check. (cherry picked from commit c9ce44b1b2d45d39ff6417a4d68bc3f4add37768) --- crates/perry-codegen/src/expr/binary.rs | 11 + .../src/expr/element_shape_guard.rs | 199 ++++-- .../src/expr/element_shape_reads.rs | 326 +++++++++ crates/perry-codegen/src/expr/mod.rs | 97 ++- .../src/expr/property_get/helpers.rs | 31 +- crates/perry-codegen/src/expr/shadow_slot.rs | 24 +- .../src/stmt/element_shape_carried.rs | 416 +++++++++++ .../stmt/element_shape_fields_random_tests.rs | 650 ++++++++++++++++++ .../src/stmt/element_shape_loop.rs | 613 ++++++++++++++--- .../src/stmt/element_shape_loop_tests.rs | 22 +- .../stmt/element_shape_shape_keyed_tests.rs | 75 +- crates/perry-codegen/src/stmt/let_stmt.rs | 2 +- crates/perry-codegen/src/stmt/mod.rs | 10 + .../src/type_analysis/numeric.rs | 23 + 14 files changed, 2297 insertions(+), 202 deletions(-) create mode 100644 crates/perry-codegen/src/expr/element_shape_reads.rs create mode 100644 crates/perry-codegen/src/stmt/element_shape_carried.rs create mode 100644 crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs diff --git a/crates/perry-codegen/src/expr/binary.rs b/crates/perry-codegen/src/expr/binary.rs index 04be6b3799..ce24554dc7 100644 --- a/crates/perry-codegen/src/expr/binary.rs +++ b/crates/perry-codegen/src/expr/binary.rs @@ -504,6 +504,17 @@ fn reduction_add_is_reassociable(ctx: &FnCtx<'_>, left: &Expr, right: &Expr) -> } fn lower_arithmetic_operand(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<(String, bool)> { + // #10199: the element-shape fast clone's two NON-numeric reads — + // `arr[i].name.length` and `arr[i].active ? 1 : 0`. Both are numbers by the + // time they get here, each proven from the loaded word's own NaN-box tag + // with a side exit to the slow clone on a miss, so the `true` suppresses + // the residual `js_number_coerce` below: that call would not slow the clone + // down, it would delete it. Outside a clone the fact vector is empty and + // this is a no-op. See `expr::element_shape_reads`. + if let Some(value) = crate::expr::element_shape_reads::try_lower_cloned_read_operand(ctx, expr)? + { + return Ok((value, true)); + } // A stable-packed numeric clone has a stronger fact than the generic // untyped-local typed-array probe below: its preheader scanned the exact // range, and its emitted-IR gate proved that no call can invalidate that diff --git a/crates/perry-codegen/src/expr/element_shape_guard.rs b/crates/perry-codegen/src/expr/element_shape_guard.rs index d2ec6ce903..7b199c3951 100644 --- a/crates/perry-codegen/src/expr/element_shape_guard.rs +++ b/crates/perry-codegen/src/expr/element_shape_guard.rs @@ -538,13 +538,137 @@ pub(crate) fn emit_element_shape_loop_preheader_check( }) } -/// Emit one `arr[i].field` read inside the fast clone: bare element load, -/// an optional residual per-element check, then a bare raw-f64 slot load. +/// Materialize this iteration's element index as an i32, or report that the +/// clone cannot index at all. /// -/// `idx_i32` must be an index the preheader discharged a bounds obligation for -/// (see [`ElementShapeIndexBound`]), and `fact` must be the fact the preheader -/// installed for this array. -pub(crate) fn emit_element_shape_field_load( +/// One arm per [`super::ElementShapeIndex`] spelling, and each reads exactly +/// the storage whose bounds obligation the preheader discharged +/// ([`ElementShapeIndexBound`]). `None` means the counter's canonical i32 slot +/// the matcher required has gone missing — that costs the read its fast +/// lowering, never correctness, and the matcher, the fact lookup and this must +/// all agree (`ElementShapeIndex::needs_counter_i32_slot`). +pub(crate) fn emit_element_shape_index( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, +) -> Option { + match &fact.index { + super::ElementShapeIndex::Counter => { + let slot = ctx.i32_counter_slots.get(&fact.index_local_id).cloned()?; + Some(ctx.block().load(I32, &slot)) + } + super::ElementShapeIndex::Constant(k) => Some(k.to_string()), + // The derived `const d = j % m` binding's own slot, written by the + // `Let` arm in `stmt/let_stmt.rs` earlier in this same iteration. + super::ElementShapeIndex::DerivedMod { slot, .. } => { + let slot = slot.clone(); + Some(ctx.block().load(I32, &slot)) + } + // #10199: the carried recurrence's private i32 slot, written by the + // body's first statement earlier in this same iteration. The REAL + // binding slot is deliberately NOT read here — it is one iteration + // behind until the trailing write-back commits, which is exactly what + // makes a mid-iteration side exit correct. + super::ElementShapeIndex::Carried(carried) => { + let slot = carried.slot.clone(); + Some(ctx.block().load(I32, &slot)) + } + } +} + +/// The masked heap handle of `arr[idx]` for THIS iteration, with the residual +/// per-element facts discharged. +/// +/// Two sources. When the matcher installed an [`super::ElementPrefetch`] the +/// body's leading virtual binding already did the deref and the residual check +/// for the whole iteration, so this is one load of an entry alloca; otherwise +/// (#10123's original shape) the deref and the check are emitted here, once per +/// read. +pub(crate) fn emit_element_shape_element_handle( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, + idx_i32: &str, +) -> String { + if let Some(prefetch) = &fact.elem_prefetch { + let slot = prefetch.handle_slot.clone(); + return ctx.block().load(I64, &slot); + } + emit_element_deref_with_residual(ctx, fact, idx_i32) +} + +/// Bare element load plus the residual per-OBJECT facts the array-level +/// invariant deliberately does not cover. Returns the masked handle. +/// +/// Emits nothing but loads, ALU and one `cond_br` — call-free by construction, +/// which is the whole revocation argument (see the module docs). +pub(crate) fn emit_element_deref_with_residual( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, + idx_i32: &str, +) -> String { + let blk = ctx.block(); + // The element-shape invariant proved every slot in the verified prefix + // is a POINTER_TAG object of the guarded identity, so the unbox needs + // no tag test and no handle-band test — the two checks that make up the + // element-read tier. + let idx64 = blk.sext(I32, idx_i32, I64); + let slot_ptr = blk.gep(I64, &fact.elements_base, &[(I64, &idx64)]); + let elem_bits = blk.load(I64, &slot_ptr); + let elem_handle = blk.and(I64, &elem_bits, crate::nanbox::POINTER_MASK_I64); + + if fact.statically_layout_proven { + return elem_handle; + } + let elem_ptr = blk.inttoptr(I64, &elem_handle); + let load_idx = ctx.new_block("element_shape.load"); + let load_label = ctx.block_label(load_idx); + let blk = ctx.block(); + + // Residual per-OBJECT facts (see the module docs for why they cannot come + // from the runtime array-level invariant). + let hdr_ptr = blk.gep(I8, &elem_ptr, &[(I64, "-8")]); + let hdr = blk.load(I32, &hdr_ptr); + let (mask, expect) = if fact.shape_keyed { + (ELEM_HEADER_SHAPE_MASK, ELEM_HEADER_SHAPE_EXPECT) + } else { + (ELEM_HEADER_MASK, ELEM_HEADER_EXPECT) + }; + let hdr_masked = blk.and(I32, &hdr, mask); + let hdr_ok = blk.icmp_eq(I32, &hdr_masked, expect); + + // #8113: the ShapeId moved from header offset 8 to 4. + let sid_ptr = blk.gep(I8, &elem_ptr, &[(I64, "4")]); + let shape_id = blk.load(I32, &sid_ptr); + let shape_ok = blk.icmp_eq(I32, &shape_id, &fact.expected_shape_id); + + let ok = blk.and(I1, &hdr_ok, &shape_ok); + // The side exit resumes the CURRENT iteration in the slow clone; no effect + // of this iteration has committed yet (#10199: with a prefetch this is the + // ONLY residual exit in the iteration, and it precedes every store). + blk.cond_br(&ok, &load_label, &fact.side_exit_label); + ctx.current_block = load_idx; + elem_handle +} + +/// #10199: the once-per-iteration element prologue. +/// +/// Emitted by the body's leading virtual binding (`stmt/let_stmt.rs` → +/// `stmt/element_shape_loop::lower_virtual_clone_binding`) once the index for +/// this iteration is in its slot. Parks the masked handle in an entry alloca so +/// every read in the iteration is a bare offset load. +pub(crate) fn emit_element_shape_prefetch( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, + idx_i32: &str, + handle_slot: &str, +) { + let handle = emit_element_deref_with_residual(ctx, fact, idx_i32); + ctx.block().store(I64, &handle, handle_slot); +} + +/// One tracked property's RAW slot word — a NaN-boxed `JSValue` in the +/// shape-keyed arm, a raw `double` in the class-keyed one. No tag test; the +/// caller adds the one its consumer needs. +pub(crate) fn emit_element_shape_slot_load( ctx: &mut FnCtx, fact: &super::ElementShapeLoopFact, idx_i32: &str, @@ -555,54 +679,27 @@ pub(crate) fn emit_element_shape_field_load( super::ElementShapeFieldSlot::Packed(index) => (I64, index.to_string()), super::ElementShapeFieldSlot::Runtime(reg) => (I64, reg.clone()), }; - - let elem_ptr = { - let blk = ctx.block(); - // The element-shape invariant proved every slot in the verified prefix - // is a POINTER_TAG object of the guarded identity, so the unbox needs - // no tag test and no handle-band test — the two checks that make up the - // element-read tier. - let idx64 = blk.sext(I32, idx_i32, I64); - let slot_ptr = blk.gep(I64, &fact.elements_base, &[(I64, &idx64)]); - let elem_bits = blk.load(I64, &slot_ptr); - let elem_handle = blk.and(I64, &elem_bits, crate::nanbox::POINTER_MASK_I64); - let elem_ptr = blk.inttoptr(I64, &elem_handle); - - if !fact.statically_layout_proven { - let load_idx = ctx.new_block("element_shape.load"); - let load_label = ctx.block_label(load_idx); - let blk = ctx.block(); - - // Residual per-OBJECT facts (see the module docs for why they - // cannot come from the runtime array-level invariant). - let hdr_ptr = blk.gep(I8, &elem_ptr, &[(I64, "-8")]); - let hdr = blk.load(I32, &hdr_ptr); - let (mask, expect) = if fact.shape_keyed { - (ELEM_HEADER_SHAPE_MASK, ELEM_HEADER_SHAPE_EXPECT) - } else { - (ELEM_HEADER_MASK, ELEM_HEADER_EXPECT) - }; - let hdr_masked = blk.and(I32, &hdr, mask); - let hdr_ok = blk.icmp_eq(I32, &hdr_masked, expect); - - // #8113: the ShapeId moved from header offset 8 to 4. - let sid_ptr = blk.gep(I8, &elem_ptr, &[(I64, "4")]); - let shape_id = blk.load(I32, &sid_ptr); - let shape_ok = blk.icmp_eq(I32, &shape_id, &fact.expected_shape_id); - - let ok = blk.and(I1, &hdr_ok, &shape_ok); - // One branch per access. The side exit resumes the CURRENT - // iteration in the slow clone; no effect has committed yet. - blk.cond_br(&ok, &load_label, &fact.side_exit_label); - ctx.current_block = load_idx; - } - elem_ptr - }; - + let elem_handle = emit_element_shape_element_handle(ctx, fact, idx_i32); let blk = ctx.block(); + let elem_ptr = blk.inttoptr(I64, &elem_handle); let fields_base = blk.gep(I8, &elem_ptr, &[(I64, &header_skip)]); let field_ptr = blk.gep(DOUBLE, &fields_base, &[(slot_ty, &slot_value)]); - let value = blk.load(DOUBLE, &field_ptr); + blk.load(DOUBLE, &field_ptr) +} + +/// Emit one `arr[i].field` read inside the fast clone: bare element load, +/// an optional residual per-element check, then a bare raw-f64 slot load. +/// +/// `idx_i32` must be an index the preheader discharged a bounds obligation for +/// (see [`ElementShapeIndexBound`]), and `fact` must be the fact the preheader +/// installed for this array. +pub(crate) fn emit_element_shape_field_load( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, + idx_i32: &str, + field_slot: &super::ElementShapeFieldSlot, +) -> String { + let value = emit_element_shape_slot_load(ctx, fact, idx_i32, field_slot); // #10123: the shape-keyed arm's representation check. // diff --git a/crates/perry-codegen/src/expr/element_shape_reads.rs b/crates/perry-codegen/src/expr/element_shape_reads.rs new file mode 100644 index 0000000000..ce207ae373 --- /dev/null +++ b/crates/perry-codegen/src/expr/element_shape_reads.rs @@ -0,0 +1,326 @@ +//! #10199: the two NON-numeric reads the element-shape fast clone admits — +//! `arr[i].prop.length` on a string field and `arr[i].prop ? A : B` on a +//! boolean one. +//! +//! ## Why they need their own lowering at all +//! +//! Both are ordinary JavaScript that the generic lowering handles perfectly +//! well — with a call. `.length` on an untyped receiver reaches the property +//! diamond and, on a miss, `js_object_get_field_by_name_f64`; a ternary on an +//! untyped condition reaches `js_is_truthy`. Inside this clone a call is not a +//! slow path, it is a DELETED clone (`contains_gc_unsafe_call`, #7690), so the +//! benchmark's `fields` shape — +//! +//! ```text +//! sum += rows[index].id; +//! sum += rows[index].name.length; +//! sum += rows[index].active ? 1 : 0; +//! ``` +//! +//! got no clone at all: one unadmitted read in the body costs the whole loop. +//! +//! ## What each one proves before it reads +//! +//! The element-shape preheader proves WHICH inline slot holds `name` and +//! `active`; it proves nothing about what is in them. So each read tag-tests +//! the loaded word and side-exits to the slow clone when the answer is not the +//! representation it is about to assume — exactly the discipline #10123 +//! established for the Number case. A `name` that is a number, a `null` or an +//! object, and an `active` that is anything but `true`/`false`, are re-run in +//! the slow clone, where full JS semantics apply. +//! +//! **`.length`.** JS `.length` is UTF-16 code units. A heap string keeps that +//! count in `StringHeader::utf16_len`, the leading `u32` — the identical load +//! the inline `.length` fast path in `property_get/generic_dispatch.rs` emits. +//! An SSO immediate (`SHORT_STRING_TAG`, up to five bytes packed into the +//! NaN-box) keeps its BYTE length in bits 40..=47, and this reads it the same +//! way the runtime's own SSO `.length` arms do +//! (`string/char_ops.rs::string_property_get_miss`, +//! `property_get/generic_dispatch.rs`). That convention is byte length, not +//! code-unit length, so a non-ASCII SSO string reports its UTF-8 size — a +//! PRE-EXISTING Perry-wide answer, reproduced here deliberately: the clone must +//! agree with the path it is a clone of, and diverging from it would be a +//! miscompile even where the shared answer is itself wrong. +//! +//! **The ternary.** JS truthiness of an arbitrary value is a runtime question +//! (`""`, `0`, `NaN`, `null`, every object). The clone does not guess it: only +//! the two boolean singletons are admitted, by exact NaN-box bit pattern, and +//! everything else side-exits. The result is one `select` between two +//! compile-time constants. + +use anyhow::Result; +use perry_hir::Expr; + +use super::FnCtx; +use crate::types::{DOUBLE, I1, I32, I64}; + +/// `STRING_TAG >> 48` — a heap `StringHeader` pointer in the low 48 bits. +const STRING_TAG_TOP16: &str = crate::nanbox::STRING_TAG_TOP16_I64; +/// `SHORT_STRING_TAG >> 48` — an SSO immediate. +const SHORT_STRING_TAG_TOP16: &str = crate::nanbox::SHORT_STRING_TAG_TOP16_I64; +/// `SHORT_STRING_LEN_SHIFT` — the length byte sits at bits 40..=47. +const SHORT_STRING_LEN_SHIFT: &str = "40"; + +/// `TAG_TRUE` (`0x7FFC_0000_0000_0004`) as a decimal i64 literal. +const TAG_TRUE_I64: &str = "9222246136947933188"; +/// `TAG_FALSE` (`0x7FFC_0000_0000_0003`) as a decimal i64 literal. +const TAG_FALSE_I64: &str = "9222246136947933187"; + +/// Is `expr` a tracked element read the fast clone has a fact for? +/// +/// Only meaningful INSIDE the clone — outside one the fact vector is empty and +/// every answer is `None`. +fn tracked_element_read(ctx: &FnCtx<'_>, expr: &Expr) -> bool { + let Expr::PropertyGet { + object, property, .. + } = expr + else { + return false; + }; + super::element_shape_loop_fact_for_property_get(ctx, object, property).is_some() +} + +/// `.length` — returns the inner read. +pub(crate) fn cloned_string_length_read<'e>(ctx: &FnCtx<'_>, expr: &'e Expr) -> Option<&'e Expr> { + let Expr::PropertyGet { + object, property, .. + } = expr + else { + return None; + }; + if property != "length" { + return None; + } + tracked_element_read(ctx, object.as_ref()).then(|| object.as_ref()) +} + +/// ` ? : ` — returns the +/// condition's inner read and the two constants. +pub(crate) fn cloned_bool_select_read<'e>( + ctx: &FnCtx<'_>, + expr: &'e Expr, +) -> Option<(&'e Expr, i64, i64)> { + let Expr::Conditional { + condition, + then_expr, + else_expr, + } = expr + else { + return None; + }; + if !tracked_element_read(ctx, condition.as_ref()) { + return None; + } + Some(( + condition.as_ref(), + integer_arm(then_expr)?, + integer_arm(else_expr)?, + )) +} + +/// A ternary arm the clone may bake in: an integer-valued literal. +/// +/// Deliberately narrow. A non-constant arm would have to be lowered on both +/// sides of a branch inside a clone whose whole admission argument is that its +/// body is one straight line of pure reads. +pub(crate) fn integer_arm(expr: &Expr) -> Option { + match expr { + Expr::Integer(k) => Some(*k), + Expr::Number(n) if n.is_finite() && n.fract() == 0.0 && *n >= -1e15 && *n <= 1e15 => { + Some(*n as i64) + } + _ => None, + } +} + +/// Resolve one tracked read to (fact, slot, index) and emit the raw slot word. +fn emit_tracked_slot_word(ctx: &mut FnCtx<'_>, read: &Expr) -> Option { + let Expr::PropertyGet { + object, property, .. + } = read + else { + return None; + }; + let (fact, slot) = super::element_shape_loop_fact_for_property_get(ctx, object, property) + .map(|(fact, slot)| (fact.clone(), slot.clone()))?; + // A class-keyed clone's slot holds a RAW double, not a NaN-boxed value — + // there is no tag to test and these two readers would be reading a + // `double`'s bit pattern as a box. Both forms are shape-keyed-only. + if !fact.shape_keyed { + return None; + } + let idx_i32 = super::element_shape_guard::emit_element_shape_index(ctx, &fact)?; + Some(super::element_shape_guard::emit_element_shape_slot_load( + ctx, &fact, &idx_i32, &slot, + )) +} + +/// The side exit the tracked read's own fact names. +fn side_exit_label(ctx: &FnCtx<'_>, read: &Expr) -> Option { + let Expr::PropertyGet { + object, property, .. + } = read + else { + return None; + }; + super::element_shape_loop_fact_for_property_get(ctx, object, property) + .map(|(fact, _)| fact.side_exit_label.clone()) +} + +/// Emit `.length` as an f64. +pub(crate) fn lower_cloned_string_length( + ctx: &mut FnCtx<'_>, + read: &Expr, +) -> Result> { + let Some(exit) = side_exit_label(ctx, read) else { + return Ok(None); + }; + let Some(value) = emit_tracked_slot_word(ctx, read) else { + return Ok(None); + }; + + let bits = ctx.block().bitcast_double_to_i64(&value); + let top16 = ctx.block().lshr(I64, &bits, "48"); + let is_heap = ctx.block().icmp_eq(I64, &top16, STRING_TAG_TOP16); + let is_sso = ctx.block().icmp_eq(I64, &top16, SHORT_STRING_TAG_TOP16); + let is_string = ctx.block().or(I1, &is_heap, &is_sso); + + let dispatch_idx = ctx.new_block("element_shape.strlen"); + let heap_idx = ctx.new_block("element_shape.strlen.heap"); + let sso_idx = ctx.new_block("element_shape.strlen.sso"); + let done_idx = ctx.new_block("element_shape.strlen.done"); + let dispatch_label = ctx.block_label(dispatch_idx); + let heap_label = ctx.block_label(heap_idx); + let sso_label = ctx.block_label(sso_idx); + let done_label = ctx.block_label(done_idx); + + // A non-string `name` re-runs the whole iteration in the slow clone rather + // than being decoded as one. + ctx.block().cond_br(&is_string, &dispatch_label, &exit); + + ctx.current_block = dispatch_idx; + ctx.block().cond_br(&is_heap, &heap_label, &sso_label); + + // `StringHeader::utf16_len` is the leading `u32`; `safe_load_i32_from_ptr` + // keeps a sub-page handle off the load, and is itself call-free (an `icmp`, + // a `select` onto a zero-valued global, and the load). + ctx.current_block = heap_idx; + let heap_handle = ctx.block().and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + let heap_len_i32 = ctx.block().safe_load_i32_from_ptr(&heap_handle); + let heap_len = ctx.block().uitofp(I32, &heap_len_i32, DOUBLE); + let heap_end = ctx.block().label.clone(); + ctx.block().br(&done_label); + + ctx.current_block = sso_idx; + let sso_shifted = ctx.block().lshr(I64, &bits, SHORT_STRING_LEN_SHIFT); + let sso_len_byte = ctx.block().and(I64, &sso_shifted, "255"); + let sso_len = ctx.block().uitofp(I64, &sso_len_byte, DOUBLE); + let sso_end = ctx.block().label.clone(); + ctx.block().br(&done_label); + + ctx.current_block = done_idx; + let length = ctx + .block() + .phi(DOUBLE, &[(&heap_len, &heap_end), (&sso_len, &sso_end)]); + Ok(Some(length)) +} + +/// Emit ` ? A : B` as an f64 select over the two constants. +pub(crate) fn lower_cloned_bool_select( + ctx: &mut FnCtx<'_>, + read: &Expr, + on_true: i64, + on_false: i64, +) -> Result> { + let Some(exit) = side_exit_label(ctx, read) else { + return Ok(None); + }; + let Some(value) = emit_tracked_slot_word(ctx, read) else { + return Ok(None); + }; + + let bits = ctx.block().bitcast_double_to_i64(&value); + let is_true = ctx.block().icmp_eq(I64, &bits, TAG_TRUE_I64); + let is_false = ctx.block().icmp_eq(I64, &bits, TAG_FALSE_I64); + let is_bool = ctx.block().or(I1, &is_true, &is_false); + + let ok_idx = ctx.new_block("element_shape.bool"); + let ok_label = ctx.block_label(ok_idx); + // Everything that is not one of the two boolean singletons — `0`, `""`, + // `null`, an object — has a JS truthiness the clone is not allowed to + // guess, so it takes the side exit. + ctx.block().cond_br(&is_bool, &ok_label, &exit); + + ctx.current_block = ok_idx; + let then_literal = format!("{:.1}", on_true as f64); + let else_literal = format!("{:.1}", on_false as f64); + Ok(Some(ctx.block().select( + I1, + &is_true, + DOUBLE, + &then_literal, + &else_literal, + ))) +} + +/// The arithmetic-operand entry point (`expr::binary::lower_arithmetic_operand`). +/// +/// Returns the lowered f64, already representation-proven, so the caller must +/// NOT append a `js_number_coerce` — that call would delete the clone. +pub(crate) fn try_lower_cloned_read_operand( + ctx: &mut FnCtx<'_>, + expr: &Expr, +) -> Result> { + if let Some(read) = cloned_string_length_read(ctx, expr) { + let read = read.clone(); + return lower_cloned_string_length(ctx, &read); + } + if let Some((read, on_true, on_false)) = cloned_bool_select_read(ctx, expr) { + let read = read.clone(); + return lower_cloned_bool_select(ctx, &read, on_true, on_false); + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Anti-drift gate, the same one `element_shape_guard`'s constants carry: + /// every literal above reproduces a `perry-runtime::value::tags` constant, + /// and `perry-codegen` cannot share the definition. + #[test] + fn boolean_singleton_literals_match_the_runtime() { + assert_eq!(TAG_TRUE_I64, (0x7FFC_0000_0000_0004u64 as i64).to_string()); + assert_eq!(TAG_FALSE_I64, (0x7FFC_0000_0000_0003u64 as i64).to_string()); + // Neither may collide with any other singleton in the 0x7FFC namespace + // — a clone that read `undefined`/`null`/a hole as `false` would be a + // miscompile, not a slow path. + for other in [ + 0x7FFC_0000_0000_0001u64, // TAG_UNDEFINED + 0x7FFC_0000_0000_0002, // TAG_NULL + 0x7FFC_0000_0000_0010, // TAG_HOLE + 0x7FFC_0000_0000_0011, // TAG_TDZ + ] { + assert_ne!(TAG_TRUE_I64, (other as i64).to_string()); + assert_ne!(TAG_FALSE_I64, (other as i64).to_string()); + } + } + + #[test] + fn string_tag_literals_match_the_runtime() { + assert_eq!(STRING_TAG_TOP16, (0x7FFFu64).to_string()); + assert_eq!(SHORT_STRING_TAG_TOP16, (0x7FF9u64).to_string()); + assert_eq!(SHORT_STRING_LEN_SHIFT, "40"); + } + + #[test] + fn only_integral_ternary_arms_are_admitted() { + assert_eq!(integer_arm(&Expr::Integer(1)), Some(1)); + assert_eq!(integer_arm(&Expr::Number(0.0)), Some(0)); + assert_eq!(integer_arm(&Expr::Number(1.5)), None); + assert_eq!(integer_arm(&Expr::Number(f64::NAN)), None); + assert_eq!(integer_arm(&Expr::Number(f64::INFINITY)), None); + assert_eq!(integer_arm(&Expr::LocalGet(3)), None); + } +} diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 9c251f68ca..ff54c5e34b 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -2136,6 +2136,66 @@ pub(crate) enum ElementShapeIndex { /// Entry-block i32 alloca the clone writes the derived index to. slot: String, }, + /// #10199: a LOOP-CARRIED index — `let c = …;` outside the loop, + /// `c = (a*c + b) % m;` as the body's first statement, then `arr[c]` (or + /// `const index = c; arr[index]`). The benchmark's `random` mode, and the + /// shape every pseudo-random walk over a record array is written in. + /// + /// Unlike [`Self::DerivedMod`]'s `d`, `c` is a real mutable binding that is + /// LIVE AFTER THE LOOP, so the clone owes a write-back. It pays it once per + /// iteration, at the END of the iteration (`Carried::commit_slot`), which is + /// also what makes the residual side exit correct: every exit from the + /// middle of an iteration leaves the real slot holding the value it had at + /// that iteration's ENTRY, so the slow clone re-runs the update exactly + /// once. A write-back at the update site would double-apply the recurrence. + Carried(Box), +} + +/// #10199: everything [`ElementShapeIndex::Carried`] needs, boxed to keep the +/// enum small. +#[derive(Debug, Clone)] +pub(crate) struct CarriedIndex { + /// The `let` binding the recurrence advances. + pub local_id: u32, + /// `const index = c` — an optional alias the body may spell the subscript + /// through. Also virtual: its `Let` emits nothing. + pub alias_id: Option, + /// `c' = (a*c + b) % m`, folded to one affine pair by the matcher. Both + /// non-negative, and `|a| * i32::MAX + |b|` proven below 2^53 so the i64 + /// evaluation agrees with the f64 one JavaScript performs. + pub coeff_a: i64, + pub coeff_b: i64, + /// i32 SSA value of the modulus, materialized in the preheader. + pub modulus_i32: String, + /// Entry-block i32 alloca holding the live carried value inside the clone. + pub slot: String, + /// The binding's REAL (NaN-boxed double) slot, written once per iteration + /// after every side exit for that iteration has been passed. + pub commit_slot: String, +} + +/// #10199: the per-iteration element prefetch. +/// +/// `rows[index].id + rows[index].name.length + (rows[index].active ? 1 : 0)` +/// reads ONE element three times. Without this each read repeats the element +/// load AND the residual header/ShapeId check, which is three branches and +/// three redundant loads per iteration. The prologue attached to the body's +/// leading virtual binding does both once and parks the masked element handle +/// in an entry alloca; every read is then a bare offset load plus the tag test +/// its own consumer needs. +/// +/// It is also what makes a MULTI-READ body's side exit correct: every residual +/// exit now happens in the prologue, before any accumulator store has +/// committed, so resuming the iteration in the slow clone cannot double-apply +/// an earlier read. +#[derive(Debug, Clone)] +pub(crate) struct ElementPrefetch { + /// The virtual binding whose lowering emits the prologue. Exactly one + /// statement owns it, so a body with both a carried update and an alias + /// emits it once. + pub site_local_id: u32, + /// Entry-block i64 alloca holding the masked element handle. + pub handle_slot: String, } impl ElementShapeIndex { @@ -2154,7 +2214,22 @@ impl ElementShapeIndex { /// then declined would hand `is_numeric_expr` a raw-double promise the /// generic path does not keep. pub(crate) fn needs_counter_i32_slot(&self) -> bool { - !matches!(self, ElementShapeIndex::Constant(_)) + !matches!( + self, + ElementShapeIndex::Constant(_) | ElementShapeIndex::Carried(_) + ) + } + + /// #10199: the locals whose `Let` / `LocalSet` the fast clone lowers + /// VIRTUALLY for this index form. Nothing may read one of them bare inside + /// the clone, and their shadow slots must not be cleared there. + pub(crate) fn virtual_locals(&self) -> impl Iterator + '_ { + let (a, b) = match self { + ElementShapeIndex::DerivedMod { local_id, .. } => (Some(*local_id), None), + ElementShapeIndex::Carried(carried) => (Some(carried.local_id), carried.alias_id), + _ => (None, None), + }; + a.into_iter().chain(b) } } @@ -2202,6 +2277,10 @@ pub(crate) struct ElementShapeLoopFact { pub index_local_id: u32, /// #10123: how the clone computes the element index. pub index: ElementShapeIndex, + /// #10199: the shared per-iteration element deref, when the body has a + /// leading virtual binding to hang it off. `None` keeps #10123's per-read + /// deref + residual check. + pub elem_prefetch: Option, /// #10123: true when the preheader proved an exact ordinary ShapeId rather /// than a class id. The per-element residual check then drops the /// typed-layout conjunct (a parsed record's layout is @@ -2231,6 +2310,15 @@ pub(crate) struct ElementShapeLoopFact { /// raw-f64 candidates; shape-keyed ones are the preheader's live query /// results (#10123). pub fields: std::collections::BTreeMap, + /// #10199: true when the fast clone lowers a body the matcher SYNTHESIZED + /// (the K-statement accumulator fold, or the carried index's trailing + /// write-back) rather than the source body. The function-wide + /// shadow-slot-clear map is keyed by statement INDEX, so a rewritten body + /// would attribute a clear to the wrong statement; the clone is call-free, + /// so no collection can observe a slot inside it and the clears are simply + /// suppressed there (`expr::emit_shadow_slot_clear`). The slow clone lowers + /// the original body, with its original indices and its original clears. + pub synthesized_body: bool, /// #7771: the body's `const r = arr[counter]` binding, when the matcher /// admitted the element-binding form. Inside the fast clone the `Let` /// itself emits nothing (`stmt/let_stmt.rs`) and every `r.field` read @@ -2306,6 +2394,12 @@ pub(crate) fn element_shape_loop_fact_for_property_get<'f>( (ElementShapeIndex::DerivedMod { local_id, .. }, Expr::LocalGet(id)) => { *id == *local_id } + // #10199: `arr[c]` and `const index = c; arr[index]` are + // the same subscript — the alias is virtual, so both + // spellings read the carried slot the preheader bounded. + (ElementShapeIndex::Carried(carried), Expr::LocalGet(id)) => { + *id == carried.local_id || carried.alias_id == Some(*id) + } _ => false, }; if !spelled { @@ -2925,6 +3019,7 @@ mod typed_array_rmw; pub(crate) use instance_misc1::builtin_parent_reserved_class_id; pub(crate) mod class_field_inline_guard; pub(crate) mod element_shape_guard; +pub(crate) mod element_shape_reads; mod js_runtime; mod literals_vars; mod logical_collections; diff --git a/crates/perry-codegen/src/expr/property_get/helpers.rs b/crates/perry-codegen/src/expr/property_get/helpers.rs index 51a8e8dbe6..302fedd4b8 100644 --- a/crates/perry-codegen/src/expr/property_get/helpers.rs +++ b/crates/perry-codegen/src/expr/property_get/helpers.rs @@ -368,29 +368,14 @@ pub(crate) fn lower_raw_f64_class_field_get_for_number_context( // array; the report below must not re-derive it from the expression // shape, which the binding form does not carry. let arr_id = fact.array_local_id; - // The counter's canonical i32 slot is what the matcher required for - // every index form that reads the counter; without it there is nothing - // to index with. A constant index reads no counter and needs none. - let counter_slot = ctx.i32_counter_slots.get(&fact.index_local_id).cloned(); - if counter_slot.is_some() || !fact.index.needs_counter_i32_slot() { - // #10123: the index the preheader discharged a bounds obligation - // for. All three forms are i32 and in `[0, length)` by the time - // they reach the GEP, which is why the fast clone pays no per-read - // bounds test in any of them. - let idx_i32 = match &fact.index { - crate::expr::ElementShapeIndex::Counter => { - let slot = counter_slot.expect("checked above"); - ctx.block().load(I32, &slot) - } - crate::expr::ElementShapeIndex::Constant(k) => k.to_string(), - // The derived `const d = j % m` binding's own slot, written by - // the `Let` arm in `stmt/let_stmt.rs` earlier in this same - // iteration. - crate::expr::ElementShapeIndex::DerivedMod { slot, .. } => { - let slot = slot.clone(); - ctx.block().load(I32, &slot) - } - }; + // #10123: the index the preheader discharged a bounds obligation for. + // Every form is an i32 in `[0, length)` by the time it reaches the GEP, + // which is why the fast clone pays no per-read bounds test in any of + // them. `None` means the counter's canonical i32 slot the matcher + // required is missing, so there is nothing to index with. + if let Some(idx_i32) = + crate::expr::element_shape_guard::emit_element_shape_index(ctx, &fact) + { let value = crate::expr::element_shape_guard::emit_element_shape_field_load( ctx, &fact, diff --git a/crates/perry-codegen/src/expr/shadow_slot.rs b/crates/perry-codegen/src/expr/shadow_slot.rs index 4f1ff8e459..02c60d1bbe 100644 --- a/crates/perry-codegen/src/expr/shadow_slot.rs +++ b/crates/perry-codegen/src/expr/shadow_slot.rs @@ -176,18 +176,32 @@ pub(crate) fn emit_shadow_slot_clear(ctx: &mut FnCtx<'_>, slot_idx: u32) { // #10123's derived index (`const d = j % m`) is the same case for the same // reason: its `Let` emits one `srem` into a private i32 alloca, never a // shadow bind, so a lexical-death clear would be the clone's only call. + // + // #10199's carried index (`c = (a*c + b) % m`) and its optional + // `const index = c` alias are two more of the same. if ctx.element_shape_loop_facts.iter().any(|fact| { - let virtual_binding = match &fact.index { - crate::expr::ElementShapeIndex::DerivedMod { local_id, .. } => Some(*local_id), - _ => None, - }; fact.element_binding .into_iter() - .chain(virtual_binding) + .chain(fact.index.virtual_locals()) .any(|id| ctx.shadow_slot_map.get(&id) == Some(&slot_idx)) }) { return; } + // #10199: a SYNTHESIZED fast-clone body (the accumulator fold, the carried + // write-back) no longer has the statement indices the function-wide clear + // map is keyed by, so every clear reached from inside it would be + // attributed to some other statement's local. Suppress them for the fast + // clone only: it is call-free, so no collection can observe a shadow slot + // while it runs, an over-rooted stale value is rewritten like any root, and + // the slow clone — lowered from the ORIGINAL body after the facts are + // popped — keeps every clear it always had. + if ctx + .element_shape_loop_facts + .iter() + .any(|fact| fact.synthesized_body) + { + return; + } // Never-bound slot: it provably still holds its initial 0 (slots are only // written through bind/set, and every value-set site binds first), so the // clear would be a redundant `js_shadow_slot_set(idx, 0)` TLS hit. diff --git a/crates/perry-codegen/src/stmt/element_shape_carried.rs b/crates/perry-codegen/src/stmt/element_shape_carried.rs new file mode 100644 index 0000000000..d6e8700436 --- /dev/null +++ b/crates/perry-codegen/src/stmt/element_shape_carried.rs @@ -0,0 +1,416 @@ +//! #10199: the element-shape clone's LOOP-CARRIED index — the `random` access +//! shape. +//! +//! ```text +//! let cursor = 0; +//! for (let i = 0; i < count; i++) { +//! cursor = (cursor * 17 + 7) % length; // the recurrence +//! const index = cursor; // optional alias +//! sum += rows[index].id; +//! } +//! ``` +//! +//! ## What makes this different from `const d = i % m` +//! +//! `d` is dead the moment the iteration ends, so #10123 could keep it entirely +//! virtual — its `Let` writes a private i32 alloca and nothing ever reads the +//! real slot. `cursor` is a real `let` that OUTLIVES the loop, so the clone +//! owes a write-back. +//! +//! **Where that write-back goes is a correctness question, not a scheduling +//! one.** The clone's residual check side-exits by resuming the CURRENT +//! iteration in the slow clone, which re-runs the whole body — the recurrence +//! included. A write-back at the update site would therefore apply +//! `cursor = (cursor * 17 + 7) % length` twice for that iteration and every +//! subsequent index would be wrong (silently: still in bounds, still a valid +//! record, just not the one JavaScript names). So the commit is the LAST thing +//! the iteration does, after every side exit it can take: +//! +//! * mid-iteration exit ⇒ the real slot still holds the previous commit, which +//! is exactly this iteration's entry value, and the slow clone advances it +//! once; +//! * normal exit ⇒ the last iteration committed, so the real slot holds the +//! final value the code after the loop reads. +//! +//! ## Why the recurrence is evaluated in i64 +//! +//! JavaScript evaluates `(cursor * 17 + 7) % length` in doubles. `frem` on +//! aarch64 is a libm `fmod` call — inside this clone a call is not a slow path, +//! it DELETES the clone (#7690) — so the recurrence is folded to one affine +//! pair `(a, b)` and evaluated as `srem i64`. +//! +//! That is only equal to what JavaScript computes while every intermediate is +//! exactly representable as a double, so the matcher tracks the maximum +//! magnitude any sub-expression can reach over `carried ∈ [0, i32::MAX]` and +//! declines above 2^53. (Fitting i64, which the original design note asked for, +//! is necessary but NOT sufficient: `c * 1e10 + 1` fits i64 comfortably and +//! loses its low bits in the f64 JavaScript actually runs.) `a` and `b` are +//! also required non-negative, so the dividend is non-negative and `srem` +//! agrees with JS `%` — which returns a NEGATIVE remainder for a negative +//! dividend, i.e. an out-of-bounds subscript, not a slow path. + +use anyhow::Result; +use perry_hir::{BinaryOp, Expr, Stmt}; + +use super::loops::{local_bound_is_loop_invariant, local_has_readable_slot}; +use crate::expr::FnCtx; +use crate::types::{DOUBLE, I32, I64}; + +/// The largest integer magnitude an `f64` represents exactly. Above it the i64 +/// recurrence and the f64 one JavaScript runs can disagree. +const EXACT_F64_INTEGER_LIMIT: i64 = 9_007_199_254_740_992; // 2^53 + +/// The preheader materializes the entry value in `0..=i32::MAX`, so this bounds +/// every `carried` the recurrence can ever see. +const MAX_CARRIED: i64 = i32::MAX as i64; + +/// A matched `carried = % m` update. +#[derive(Clone, Copy, Debug)] +pub(super) struct MatchedCarried { + pub carried_id: u32, + pub modulus_id: u32, + pub coeff_a: i64, + pub coeff_b: i64, +} + +/// Fold `expr` to `a * carried + b`, tracking the largest magnitude ANY +/// sub-expression can reach for `carried ∈ [0, i32::MAX]`. +/// +/// Returns `(a, b, max_abs)`. `None` declines — an unknown node, a second +/// variable, a non-integral literal, or any overflow of the checked i64 +/// arithmetic used to compute the bounds. +/// +/// The magnitude is tracked per NODE rather than taken from the folded result +/// because folding can cancel: `c * 1000 - c * 999` is `1 * c`, but JavaScript +/// still evaluates both thousand-fold products. +fn affine_over_carried(expr: &Expr, carried_id: u32) -> Option<(i64, i64, i64)> { + let magnitude = |a: i64, b: i64| -> Option { + a.checked_abs()? + .checked_mul(MAX_CARRIED)? + .checked_add(b.checked_abs()?) + }; + match expr { + Expr::LocalGet(id) if *id == carried_id => Some((1, 0, MAX_CARRIED)), + Expr::Integer(k) => Some((0, *k, k.checked_abs()?)), + Expr::Number(n) if n.is_finite() && n.fract() == 0.0 => { + // `as i64` saturates rather than wrapping, so an out-of-range + // literal would fold to i64::MAX and read as a huge-but-valid + // coefficient. Range-check before converting. + if *n < -(EXACT_F64_INTEGER_LIMIT as f64) || *n > EXACT_F64_INTEGER_LIMIT as f64 { + return None; + } + let k = *n as i64; + Some((0, k, k.checked_abs()?)) + } + Expr::Binary { op, left, right } => { + let (la, lb, lmax) = affine_over_carried(left, carried_id)?; + let (ra, rb, rmax) = affine_over_carried(right, carried_id)?; + let (a, b) = match op { + BinaryOp::Add => (la.checked_add(ra)?, lb.checked_add(rb)?), + BinaryOp::Sub => (la.checked_sub(ra)?, lb.checked_sub(rb)?), + // A product is affine only when one side is a constant; `c * c` + // is not, and neither is `c * m` for a runtime `m`. + BinaryOp::Mul if la == 0 => (ra.checked_mul(lb)?, rb.checked_mul(lb)?), + BinaryOp::Mul if ra == 0 => (la.checked_mul(rb)?, lb.checked_mul(rb)?), + _ => return None, + }; + let here = magnitude(a, b)?; + Some((a, b, lmax.max(rmax).max(here))) + } + _ => None, + } +} + +/// Does `expr` read `id`? +/// +/// Conservative by construction: an expression node this does not enumerate +/// answers "yes". The alternative shape — a catch-all that returns `false` — +/// is the #6377 / `collect_local_refs_expr` failure mode CLAUDE.md names, and +/// here it would let an accumulator read be folded into a statement that no +/// longer sees the value it was written against. +pub(super) fn expr_reads_local(expr: &Expr, id: u32) -> bool { + match expr { + Expr::LocalGet(read) => *read == id, + Expr::Number(_) | Expr::Integer(_) => false, + Expr::Binary { left, right, .. } + | Expr::MathImul(left, right) + | Expr::MathPow(left, right) => expr_reads_local(left, id) || expr_reads_local(right, id), + Expr::NumberCoerce(operand) => expr_reads_local(operand, id), + Expr::MathMin(values) | Expr::MathMax(values) => { + values.iter().any(|e| expr_reads_local(e, id)) + } + Expr::MathAbs(value) + | Expr::MathSqrt(value) + | Expr::MathFloor(value) + | Expr::MathCeil(value) + | Expr::MathRound(value) + | Expr::MathTrunc(value) + | Expr::MathSign(value) + | Expr::MathF16round(value) => expr_reads_local(value, id), + Expr::PropertyGet { object, .. } => expr_reads_local(object, id), + Expr::IndexGet { object, index } => { + expr_reads_local(object, id) || expr_reads_local(index, id) + } + Expr::Conditional { + condition, + then_expr, + else_expr, + } => { + expr_reads_local(condition, id) + || expr_reads_local(then_expr, id) + || expr_reads_local(else_expr, id) + } + _ => true, + } +} + +/// Match the body's first statement as `carried = % m`. +#[allow(clippy::too_many_arguments)] +pub(super) fn match_carried_update( + ctx: &FnCtx<'_>, + stmt: &Stmt, + counter_id: u32, + condition: &Expr, + update: Option<&Expr>, + body: &[Stmt], +) -> Option { + let Stmt::Expr(Expr::LocalSet(carried_id, value)) = stmt else { + return None; + }; + let Expr::Binary { + op: BinaryOp::Mod, + left, + right, + } = value.as_ref() + else { + return None; + }; + let Expr::LocalGet(modulus_id) = right.as_ref() else { + return None; + }; + if *carried_id == counter_id || *modulus_id == counter_id || *modulus_id == *carried_id { + return None; + } + + let (coeff_a, coeff_b, max_abs) = affine_over_carried(left, *carried_id)?; + // A negative dividend makes JS `%` return a negative remainder — an + // out-of-bounds subscript the preheader's `m <= length` says nothing about. + // With `carried >= 0` (the preheader materializes it in `0..=i32::MAX`) and + // both coefficients non-negative, the dividend cannot be negative. + if coeff_a < 0 || coeff_b < 0 { + return None; + } + if max_abs > EXACT_F64_INTEGER_LIMIT { + return None; + } + + // The binding must be a plain, directly-addressable local the clone can + // write back to with one store. A module global would need the rooted-store + // path, a boxed or captured binding lives in a cell a plain store would + // leave stale for an observer outside the clone. + if !ctx.locals.contains_key(carried_id) + || ctx.module_globals.contains_key(carried_id) + || ctx.boxed_vars.contains(carried_id) + || ctx.closure_captures.contains_key(carried_id) + || !local_has_readable_slot(ctx, *carried_id) + || ctx.pod_records.contains_key(carried_id) + || ctx.scalar_replaced.contains_key(carried_id) + { + return None; + } + // `carried` is written exactly once per iteration — by the statement above. + // Every other statement the body matcher admits writes the accumulator, and + // it rejects a body whose accumulator IS the carried local, so the only way + // a second write could appear is a `for` update expression naming it. + if update.is_some_and(|u| expr_writes_local(u, *carried_id)) { + return None; + } + // The modulus is materialized ONCE in the preheader and every iteration's + // `srem` uses that value, so a body (or update, or condition) that could + // rewrite it would derive indices against a stale bound. + if ctx.boxed_vars.contains(modulus_id) + || ctx.closure_captures.contains_key(modulus_id) + || !(local_has_readable_slot(ctx, *modulus_id) + || ctx.module_globals.contains_key(modulus_id)) + || !local_bound_is_loop_invariant(condition, update, body, *modulus_id) + { + return None; + } + + Some(MatchedCarried { + carried_id: *carried_id, + modulus_id: *modulus_id, + coeff_a, + coeff_b, + }) +} + +/// Does `expr` assign to `id`? Conservative in the same direction as +/// [`expr_reads_local`] — an unenumerated node answers "yes". +fn expr_writes_local(expr: &Expr, id: u32) -> bool { + match expr { + Expr::LocalSet(target, _) => *target == id, + Expr::Update { id: target, .. } => *target == id, + Expr::LocalGet(_) | Expr::Number(_) | Expr::Integer(_) => false, + Expr::Binary { left, right, .. } => { + expr_writes_local(left, id) || expr_writes_local(right, id) + } + _ => true, + } +} + +/// Lower the fast clone's carried-index statements, or report that this +/// expression is not one. +/// +/// Two shapes, both synthesized or admitted by the matcher, and neither lowered +/// generically: +/// +/// * the UPDATE `carried = % m` — one `mul`/`add`/`srem` chain in i64 +/// writing the clone's private i32 slot, never the real binding; +/// * the COMMIT `carried = carried`, the trailing statement the matcher appends +/// to the synthesized fast body — one `sitofp` + `store` that publishes the +/// iteration's value to the real slot. +/// +/// The commit is spelled as a self-assignment because it has to be a statement +/// the generic lowering would also accept (the slow clone never sees it — the +/// facts are popped first — but nothing may depend on that). A user-written +/// `c = c` inside a body cannot reach here: the body matcher admits only the +/// carried update, one virtual binding and accumulator writes, and `c` is not +/// the accumulator. +pub(super) fn lower_virtual_carried_stmt(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { + let Expr::LocalSet(id, value) = expr else { + return Ok(false); + }; + let Some((carried, scope_id)) = ctx.element_shape_loop_facts.iter().rev().find_map(|fact| { + let crate::expr::ElementShapeIndex::Carried(carried) = &fact.index else { + return None; + }; + (carried.local_id == *id).then(|| (carried.clone(), fact.scope_id)) + }) else { + return Ok(false); + }; + + if matches!(value.as_ref(), Expr::LocalGet(read) if *read == *id) { + // COMMIT. The value published here is the one every read in this + // iteration already used, so the slot and the binding agree from the + // end of the iteration onward. + let slot = carried.slot.clone(); + let commit_slot = carried.commit_slot.clone(); + let blk = ctx.block(); + let current = blk.load(I32, &slot); + let boxed = blk.sitofp(I32, ¤t, DOUBLE); + blk.store(DOUBLE, &boxed, &commit_slot); + return Ok(true); + } + + // UPDATE. `0 <= carried <= i32::MAX` and `1 <= m <= i32::MAX` hold by the + // preheader's two materializations, and the matcher proved + // `a * i32::MAX + b < 2^53`, so the i64 chain cannot overflow and agrees + // with the f64 arithmetic JavaScript performs. + let slot = carried.slot.clone(); + let modulus_i32 = carried.modulus_i32.clone(); + let coeff_a = carried.coeff_a.to_string(); + let coeff_b = carried.coeff_b.to_string(); + { + let blk = ctx.block(); + let current = blk.load(I32, &slot); + let current64 = blk.sext(I32, ¤t, I64); + let scaled = blk.mul(I64, ¤t64, &coeff_a); + let shifted = blk.add(I64, &scaled, &coeff_b); + let modulus64 = blk.sext(I32, &modulus_i32, I64); + let wrapped = blk.srem(I64, &shifted, &modulus64); + let next = blk.trunc(I64, &wrapped, I32); + blk.store(I32, &next, &slot); + } + // The index for this iteration is now in its slot, so a body with no alias + // binding hangs the shared element prologue off the update itself. + super::element_shape_loop::emit_element_prefetch_for(ctx, scope_id, *id); + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn local(id: u32) -> Expr { + Expr::LocalGet(id) + } + fn mul(l: Expr, r: Expr) -> Expr { + Expr::Binary { + op: BinaryOp::Mul, + left: Box::new(l), + right: Box::new(r), + } + } + fn add(l: Expr, r: Expr) -> Expr { + Expr::Binary { + op: BinaryOp::Add, + left: Box::new(l), + right: Box::new(r), + } + } + fn sub(l: Expr, r: Expr) -> Expr { + Expr::Binary { + op: BinaryOp::Sub, + left: Box::new(l), + right: Box::new(r), + } + } + + #[test] + fn the_benchmark_recurrence_folds() { + // `(cursor * 17 + 7)` + let e = add(mul(local(9), Expr::Integer(17)), Expr::Integer(7)); + assert_eq!( + affine_over_carried(&e, 9), + Some((17, 7, 17 * MAX_CARRIED + 7)) + ); + } + + #[test] + fn a_constant_only_expression_folds_to_b() { + assert_eq!(affine_over_carried(&Expr::Integer(5), 9), Some((0, 5, 5))); + } + + #[test] + fn a_second_variable_declines() { + assert_eq!(affine_over_carried(&add(local(9), local(10)), 9), None); + assert_eq!(affine_over_carried(&mul(local(9), local(9)), 9), None); + } + + /// The magnitude bound is per NODE, not of the folded pair: cancellation + /// must not hide an intermediate JavaScript really computes in f64. + #[test] + fn cancelling_products_still_report_their_intermediates() { + // `c * 1000000000 - c * 999999999` folds to `1 * c`, but both products + // reach ~2e18 for a large `c` — far past 2^53, where the f64 JS runs + // and this i64 chain stop agreeing. + let e = sub( + mul(local(9), Expr::Integer(1_000_000_000)), + mul(local(9), Expr::Integer(999_999_999)), + ); + let (a, b, max_abs) = affine_over_carried(&e, 9).expect("affine"); + assert_eq!((a, b), (1, 0)); + assert!( + max_abs > EXACT_F64_INTEGER_LIMIT, + "the intermediate magnitude must be reported, not the folded one" + ); + } + + #[test] + fn a_non_integral_literal_declines() { + assert_eq!(affine_over_carried(&Expr::Number(1.5), 9), None); + assert_eq!(affine_over_carried(&Expr::Number(f64::NAN), 9), None); + assert_eq!(affine_over_carried(&Expr::Number(f64::INFINITY), 9), None); + // Saturating `as i64` would turn this into a plausible coefficient. + assert_eq!(affine_over_carried(&Expr::Number(1e300), 9), None); + } + + #[test] + fn reads_are_detected_conservatively() { + assert!(expr_reads_local(&add(local(9), Expr::Integer(1)), 9)); + assert!(!expr_reads_local(&add(local(8), Expr::Integer(1)), 9)); + // An unenumerated node answers "reads it". + assert!(expr_reads_local(&Expr::This, 9)); + } +} diff --git a/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs b/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs new file mode 100644 index 0000000000..c86fbd7e88 --- /dev/null +++ b/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs @@ -0,0 +1,650 @@ +//! #10199: the `fields` and `random` access shapes — IR census for the +//! element-shape clone's loop-carried index, its multi-statement accumulator +//! fold, and its two non-numeric reads. +//! +//! A child module of `element_shape_shape_keyed_tests` (see the `mod` +//! declaration there), so `use super::*` brings both that file's shape-keyed +//! assertions and `element_shape_loop_tests`'s slicing helpers. +//! +//! Every positive here is paired with a sabotage case, and the two liveness +//! assertions #10171 was short — the preheader's EXACT comparison set, and that +//! the residual check really is shared rather than repeated — are the ones that +//! would have caught a clone that is emitted, branched into, and never entered +//! or never actually cheaper. + +use super::*; + +const ROWS2_ID: u32 = 40; +const COUNT2_ID: u32 = 41; +const MOD2_ID: u32 = 42; +const SUM2_ID: u32 = 43; +const COUNTER2_ID: u32 = 44; +const CURSOR_ID: u32 = 45; +const ALIAS_ID: u32 = 46; +const OTHER_ID: u32 = 47; + +/// `rows[].` +fn rows_field(index: Expr, prop: &str) -> Expr { + Expr::PropertyGet { + object: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ROWS2_ID)), + index: Box::new(index), + }), + property: prop.to_string(), + byte_offset: 0, + } +} + +/// `sum = sum + ` +fn add_to_sum(value: Expr) -> Stmt { + Stmt::Expr(Expr::LocalSet( + SUM2_ID, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(SUM2_ID)), + right: Box::new(value), + }), + )) +} + +fn binary(op: BinaryOp, left: Expr, right: Expr) -> Expr { + Expr::Binary { + op, + left: Box::new(left), + right: Box::new(right), + } +} + +/// `cursor = % n` +fn cursor_update(affine: Expr) -> Stmt { + Stmt::Expr(Expr::LocalSet( + CURSOR_ID, + Box::new(binary(BinaryOp::Mod, affine, Expr::LocalGet(MOD2_ID))), + )) +} + +/// The benchmark's own recurrence, `(cursor * 17 + 7)`. +fn benchmark_affine() -> Expr { + binary( + BinaryOp::Add, + binary(BinaryOp::Mul, Expr::LocalGet(CURSOR_ID), Expr::Integer(17)), + Expr::Integer(7), + ) +} + +/// `const index = cursor;` +fn cursor_alias() -> Stmt { + Stmt::Let { + id: ALIAS_ID, + name: "index".to_string(), + ty: Type::Number, + mutable: false, + init: Some(Expr::LocalGet(CURSOR_ID)), + } +} + +/// `const index = i % n;` +fn modulo_index() -> Stmt { + Stmt::Let { + id: ALIAS_ID, + name: "index".to_string(), + ty: Type::Any, + mutable: false, + init: Some(binary( + BinaryOp::Mod, + Expr::LocalGet(COUNTER2_ID), + Expr::LocalGet(MOD2_ID), + )), + } +} + +/// ```text +/// function run(rows: any, count: number, n: number): number { +/// let sum = 0; +/// let cursor = 0; +/// for (let i = 0; i < count; i++) +/// return sum; +/// } +/// ``` +fn access_module(body: Vec) -> Module { + let mut m = Module::new("element_shape_loop.ts"); + let param = |id: u32, name: &str, ty: Type| perry_hir::Param { + id, + name: name.to_string(), + ty, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }; + m.functions = vec![perry_hir::Function { + id: 902, + name: "run".to_string(), + type_params: Vec::new(), + params: vec![ + param(ROWS2_ID, "rows", Type::Any), + param(COUNT2_ID, "count", Type::Number), + param(MOD2_ID, "n", Type::Number), + ], + return_type: Type::Number, + body: vec![ + Stmt::Let { + id: SUM2_ID, + name: "sum".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + }, + Stmt::Let { + id: CURSOR_ID, + name: "cursor".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + }, + Stmt::For { + init: Some(Box::new(Stmt::Let { + id: COUNTER2_ID, + name: "i".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + })), + condition: Some(Expr::Compare { + op: CompareOp::Lt, + left: Box::new(Expr::LocalGet(COUNTER2_ID)), + right: Box::new(Expr::LocalGet(COUNT2_ID)), + }), + update: Some(Expr::Update { + id: COUNTER2_ID, + op: UpdateOp::Increment, + prefix: false, + }), + body, + }, + Stmt::Return(Some(Expr::LocalGet(SUM2_ID))), + ], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }]; + m.init_kind = ModuleInitKind::Eager; + m +} + +/// The benchmark's `random` body, with and without the alias spelling. +fn random_body(with_alias: bool) -> Vec { + let index = if with_alias { ALIAS_ID } else { CURSOR_ID }; + let mut body = vec![cursor_update(benchmark_affine())]; + if with_alias { + body.push(cursor_alias()); + } + body.push(add_to_sum(rows_field(Expr::LocalGet(index), "id"))); + body +} + +/// The benchmark's `fields` body: three accumulator statements over ONE +/// element, one through a string and one through a boolean. +fn fields_body() -> Vec { + vec![ + modulo_index(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + add_to_sum(Expr::PropertyGet { + object: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "name")), + property: "length".to_string(), + byte_offset: 0, + }), + add_to_sum(Expr::Conditional { + condition: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "active")), + then_expr: Box::new(Expr::Integer(1)), + else_expr: Box::new(Expr::Integer(0)), + }), + ] +} + +/// How many side exits the clone can take per iteration — one branch per +/// residual / tag test that leaves for the slow preheader. +fn side_exit_count(fast: &str) -> usize { + fast.matches("label %element_shape.loop.slow.preheader") + .count() +} + +/// The LAST block in `fast` whose label starts with `prefix`, body only. +/// +/// Used to name the block a commit must live in. Reaching a block named after a +/// passed tag test IS the proof that every side exit of the iteration is behind +/// it — which is stronger than any textual before/after comparison, because the +/// emitted block ORDER is not the execution order (the loop's `update` block is +/// printed before the body blocks that branch into it). +fn last_block_with_prefix<'a>(fast: &'a str, prefix: &str) -> &'a str { + let start = fast + .rmatch_indices('\n') + .map(|(idx, _)| idx + 1) + .find(|&idx| { + fast[idx..].starts_with(prefix) + && fast[idx..] + .lines() + .next() + .unwrap_or("") + .trim_end() + .ends_with(':') + }) + .unwrap_or_else(|| panic!("no block named `{prefix}*` in:\n{fast}")); + let body = &fast[start..]; + let end = body.find("\n\n").unwrap_or(body.len()); + &body[..end] +} + +/// Count the `store` instructions one iteration of the clone executes. +fn store_count(fast: &str) -> usize { + fast.lines() + .filter(|l| l.trim_start().starts_with("store ")) + .count() +} + +// --------------------------------------------------------------------------- +// #10199 — the LOOP-CARRIED index (`random`). +// --------------------------------------------------------------------------- + +#[test] +fn a_carried_recurrence_gets_a_shape_keyed_clone() { + let ir = emit(&access_module(random_body(true))); + assert_shape_keyed_clone(&ir, "carried index with an alias"); + let fast = fast_clone_slice(&ir); + assert!( + fast.contains("srem i64") && fast.contains("mul i64"), + "the recurrence must be the i64 chain, not an `frem` (a libm call on \ + aarch64, which would delete the clone); emitted:\n{fast}" + ); + assert!( + !fast.contains("frem"), + "an `frem` in the clone is the whole reason this form needed its own \ + lowering; emitted:\n{fast}" + ); + assert!( + ir.contains("element_shape.loop.carried.range"), + "the carried ENTRY value must be materialized as a validated \ + non-negative i32 — the i64 bound and the `srem` sign argument are \ + both stated over `0..=i32::MAX`" + ); + assert!( + ir.contains("element_shape.loop.modulus.range"), + "the modulus must be materialized: `srem` by zero is UB and `x % 0` \ + is NaN in JS" + ); +} + +#[test] +fn a_carried_recurrence_matches_without_the_alias() { + // `rows[cursor]` directly — the alias is sugar, not part of the form. + let ir = emit(&access_module(random_body(false))); + assert_shape_keyed_clone(&ir, "carried index, no alias"); + let fast = fast_clone_slice(&ir); + assert!(fast.contains("srem i64"), "emitted:\n{fast}"); +} + +/// THE correctness assertion for this form. +/// +/// The residual side exit resumes the CURRENT iteration in the slow clone, +/// which re-runs the whole body — the recurrence included. So the write-back to +/// the real binding must come after EVERY exit the iteration can take, or the +/// recurrence is applied twice and every later index is silently wrong (still +/// in bounds, still a valid record, just not the one JavaScript names). +#[test] +fn the_carried_write_back_follows_every_side_exit() { + let ir = emit(&access_module(random_body(true))); + let fast = fast_clone_slice(&ir); + // Every side exit this iteration can take is a tag test on a loaded word, + // and the LAST of them is the `id` read's Number test. Its admitted arm — + // `element_shape.number.*` — is therefore reachable only once the whole + // iteration is past every exit, so a commit inside it is provably last. + let committed = last_block_with_prefix(&fast, "element_shape.number"); + assert!( + committed.contains("sitofp i32") && committed.contains("store double"), + "the carried write-back must live past every side exit; a write-back \ + before one makes the slow clone apply the recurrence twice for that \ + iteration, and every later index is silently wrong. \ + final block:\n{committed}\nfull clone:\n{fast}" + ); + assert_eq!( + committed.matches("store double").count(), + 1, + "the carried value must be published exactly once per iteration; \ + final block:\n{committed}" + ); + assert_eq!( + store_count(&fast), + // the carried i32 slot, the prefetched element handle, the + // accumulator, the carried write-back, and the counter's increment + 5, + "a second write to the carried binding would double-apply the \ + recurrence on a side exit; emitted:\n{fast}" + ); +} + +/// The same liveness assertion #10123's derived index carries: the ONE length +/// comparison the preheader owes is `modulus <= length`, NOT the counter arm's +/// `length >= bound`. Demanding the latter makes the clone unenterable whenever +/// the loop runs more times than the array is long — which is every access +/// benchmark. +#[test] +fn a_carried_index_owes_exactly_the_modulus_obligation() { + let ir = emit(&access_module(random_body(true))); + let deref = block_slice(&ir, "element_shape.loop.preheader.deref"); + assert_eq!( + deref.matches("icmp uge i32").count(), + 1, + "a carried index owes exactly ONE length comparison (`modulus <= \ + length`); emitted:\n{deref}" + ); + let fast = fast_clone_slice(&ir); + assert!( + !fast.contains("icmp ult i32") && !fast.contains("icmp ugt i32"), + "the bounds obligation is discharged ONCE in the preheader; a per-read \ + test in the clone would mean it was not; emitted:\n{fast}" + ); +} + +#[test] +fn a_negative_coefficient_declines() { + // `(cursor * 17 - 7) % n` can produce a negative dividend, and JS `%` + // returns a NEGATIVE remainder for one — an out-of-bounds subscript the + // preheader's `m <= length` says nothing about. + let body = vec![ + cursor_update(binary( + BinaryOp::Sub, + binary(BinaryOp::Mul, Expr::LocalGet(CURSOR_ID), Expr::Integer(17)), + Expr::Integer(7), + )), + cursor_alias(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a recurrence whose dividend can go negative must decline" + ); +} + +#[test] +fn a_non_affine_recurrence_declines() { + // `(cursor * cursor) % n` is not affine, so no (a, b) pair describes it and + // no magnitude bound can be proven. + let body = vec![ + cursor_update(binary( + BinaryOp::Mul, + Expr::LocalGet(CURSOR_ID), + Expr::LocalGet(CURSOR_ID), + )), + cursor_alias(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a non-affine recurrence must decline" + ); +} + +#[test] +fn a_recurrence_that_leaves_exact_double_range_declines() { + // `cursor * 1e9 + 1` fits i64 comfortably and does NOT fit the f64 that + // JavaScript actually evaluates it in, so the i64 chain and the program + // would disagree. "Fits i64" was the original design note's obligation; it + // is necessary and not sufficient. + let body = vec![ + cursor_update(binary( + BinaryOp::Add, + binary( + BinaryOp::Mul, + Expr::LocalGet(CURSOR_ID), + Expr::Integer(1_000_000_000), + ), + Expr::Integer(1), + )), + cursor_alias(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a recurrence whose intermediates pass 2^53 must decline: JS evaluates \ + it in doubles and the clone's i64 chain would not agree" + ); +} + +#[test] +fn a_bare_read_of_the_carried_local_declines() { + // Inside the clone the real slot is one iteration behind until the trailing + // commit, so a bare `cursor` in the accumulator would read a stale value. + let body = vec![ + cursor_update(benchmark_affine()), + cursor_alias(), + add_to_sum(binary( + BinaryOp::Add, + rows_field(Expr::LocalGet(ALIAS_ID), "id"), + Expr::LocalGet(CURSOR_ID), + )), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a bare read of the carried local must decline the clone" + ); +} + +#[test] +fn a_recurrence_over_a_second_variable_declines() { + // `(cursor + other) % n` is not a function of `cursor` alone; `other` is + // not materialized, bounded, or proven loop-invariant. + let body = vec![ + cursor_update(binary( + BinaryOp::Add, + Expr::LocalGet(CURSOR_ID), + Expr::LocalGet(OTHER_ID), + )), + cursor_alias(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a recurrence reading a second variable must decline" + ); +} + +// --------------------------------------------------------------------------- +// #10199 — the MULTI-STATEMENT body, the string `.length` and the ternary +// (`fields`). +// --------------------------------------------------------------------------- + +#[test] +fn the_fields_body_gets_a_shape_keyed_clone() { + let ir = emit(&access_module(fields_body())); + assert_shape_keyed_clone(&ir, "three reads of one element"); + let fast = fast_clone_slice(&ir); + assert!( + fast.contains("element_shape.strlen.heap") && fast.contains("element_shape.strlen.sso"), + "a string `.length` must decode BOTH representations inline — a call \ + to the runtime would delete the clone; emitted:\n{fast}" + ); + assert!( + fast.contains("select i1"), + "the ternary must become a select between the two constants; \ + emitted:\n{fast}" + ); + assert!( + fast.contains("9222246136947933188") && fast.contains("9222246136947933187"), + "the ternary must test the two boolean singletons by exact NaN-box bit \ + pattern; guessing JS truthiness in the clone would be a miscompile; \ + emitted:\n{fast}" + ); +} + +/// THE cost assertion. Three reads of `rows[index]` are three loads of the same +/// element and three repeats of the residual header/ShapeId check unless the +/// prologue shares them — and a clone that is entered but pays 3x the guard is +/// exactly the "emitted, branched into, and not actually cheaper" failure the +/// IR census exists to catch. +#[test] +fn three_reads_of_one_element_share_one_residual_check() { + let ir = emit(&access_module(fields_body())); + let fast = fast_clone_slice(&ir); + assert_eq!( + fast.matches("134250751").count(), + 1, + "the shape-keyed residual mask must appear ONCE per iteration, not \ + once per read; emitted:\n{fast}" + ); + assert_eq!( + fast.matches("element_shape.load").count(), + // one block definition plus the `br` that targets it + 2, + "there must be exactly one residual-check block in the iteration; \ + emitted:\n{fast}" + ); + // Four tag/residual tests, not twelve: the residual once, then one + // representation test per read. + assert_eq!( + side_exit_count(&fast), + 4, + "one residual check plus one tag test per read; emitted:\n{fast}" + ); +} + +/// THE correctness assertion for the multi-statement body. `sum += a; sum += b` +/// commits `sum` twice; if the second read's tag test side-exits, the slow +/// clone re-runs the iteration and applies `a` a second time. The fold makes +/// the whole iteration commit once, after every exit. +#[test] +fn the_accumulator_commits_once_after_every_side_exit() { + let ir = emit(&access_module(fields_body())); + let fast = fast_clone_slice(&ir); + assert_eq!( + fast.matches("fadd double").count(), + 3, + "the fold keeps all three additions in source order — it changes when \ + the result is STORED, never what is computed; emitted:\n{fast}" + ); + assert_eq!( + store_count(&fast), + // the derived index, the prefetched element handle, the accumulator, + // and the counter's own increment + 4, + "three accumulator statements must fold to ONE accumulator store; a \ + second one means a side exit from a later read can leave an earlier \ + read already applied when the slow clone re-runs the iteration; \ + emitted:\n{fast}" + ); + // The one store lives in the block the LAST tag test branches into, so it + // is behind every side exit of the iteration. + let committed = last_block_with_prefix(&fast, "element_shape.bool"); + assert!( + committed.contains("store "), + "the accumulator store must live past every side exit; final \ + block:\n{committed}\nfull clone:\n{fast}" + ); +} + +#[test] +fn an_addend_that_reads_the_accumulator_declines() { + // `sum += rows[i].id; sum += sum` — folding would substitute the PRE-first + // -statement value of `sum` into the second addend. + let body = vec![ + modulo_index(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + add_to_sum(Expr::LocalGet(SUM2_ID)), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "an addend reading the accumulator must decline the fold, and with it \ + the clone" + ); +} + +#[test] +fn two_accumulators_decline() { + let body = vec![ + modulo_index(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + Stmt::Expr(Expr::LocalSet( + CURSOR_ID, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(CURSOR_ID)), + right: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + }), + )), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "the body must write exactly one accumulator" + ); +} + +#[test] +fn a_ternary_with_a_non_constant_arm_declines() { + let body = vec![ + modulo_index(), + add_to_sum(Expr::Conditional { + condition: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "active")), + then_expr: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + else_expr: Box::new(Expr::Integer(0)), + }), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "only integer-constant ternary arms are admitted" + ); +} + +#[test] +fn a_ternary_on_something_other_than_an_element_read_declines() { + let body = vec![ + modulo_index(), + add_to_sum(Expr::Conditional { + condition: Box::new(Expr::LocalGet(COUNTER2_ID)), + then_expr: Box::new(Expr::Integer(1)), + else_expr: Box::new(Expr::Integer(0)), + }), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "the clone lowers a boolean-singleton test, not general JS truthiness" + ); +} + +/// The two non-numeric reads are SHAPE-keyed only: a class-keyed clone's slot +/// holds a raw `double`, so there is no NaN-box tag to test and both readers +/// would be decoding a double's bit pattern. +#[test] +fn a_class_typed_array_declines_the_string_length_read() { + let ir = emit(&element_shape_module( + vec![Stmt::Expr(Expr::LocalSet( + SUM_ID, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(SUM_ID)), + right: Box::new(Expr::PropertyGet { + object: Box::new(elem_field(ARRAY_ID, Expr::LocalGet(COUNTER_ID))), + property: "length".to_string(), + byte_offset: 0, + }), + }), + ))], + None, + )); + assert!( + !ir.contains("element_shape.strlen"), + "the class-keyed arm reads raw doubles; there is no tag to test" + ); +} diff --git a/crates/perry-codegen/src/stmt/element_shape_loop.rs b/crates/perry-codegen/src/stmt/element_shape_loop.rs index 1ce4f0cd8d..9bc43a8d17 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop.rs @@ -152,7 +152,7 @@ use super::loops::{ CLASS_FIELD_LOOP_PROP_DENYLIST, }; use crate::expr::{lower_expr, FnCtx}; -use crate::types::{DOUBLE, I1, I32}; +use crate::types::{DOUBLE, I1, I32, I64}; /// Loop bound: a literal, a loop-invariant local / module global that is /// materialized to i32 once in the preheader, or the tracked array's own @@ -210,14 +210,28 @@ enum ElementShapeIdentity { enum MatchedIndex { Counter, Constant(i64), - DerivedMod { local_id: u32, modulus_id: u32 }, + DerivedMod { + local_id: u32, + modulus_id: u32, + }, + /// #10199: `c = (a*c + b) % m; … arr[c]`, optionally spelled through a + /// `const index = c` alias. The recurrence itself lives on + /// [`ElementShapeVersionedLoop::carried`] — this arm records only the + /// spellings the subscript may take. See `stmt/element_shape_carried.rs`. + Carried { + carried_id: u32, + alias_id: Option, + }, } impl MatchedIndex { /// Mirror of [`crate::expr::ElementShapeIndex::needs_counter_i32_slot`], /// asked before the fact exists. fn needs_counter_i32_slot(self) -> bool { - !matches!(self, MatchedIndex::Constant(_)) + !matches!( + self, + MatchedIndex::Constant(_) | MatchedIndex::Carried { .. } + ) } } @@ -235,6 +249,21 @@ struct ElementShapeVersionedLoop { /// form; `None` for the original single-statement accumulator body. element_binding: Option, accumulator_id: u32, + /// #10199: the matched recurrence, for the [`MatchedIndex::Carried`] form. + carried: Option, + /// #10199: the body the FAST clone lowers, when it is not the source body. + /// + /// Two rewrites, both of which exist to put every side exit an iteration + /// can take BEFORE any of its stores: + /// + /// * K accumulator statements fold into one, so a tag test that fails on + /// the third read cannot leave the first two already applied to `acc` + /// when the slow clone re-runs the iteration; + /// * a carried recurrence gains a trailing commit statement, so the real + /// binding is published once the iteration is past every exit. + /// + /// The SLOW clone always lowers the original `body`. + fast_body: Option>, } /// The locals the pure-expression walk reasons about. @@ -246,6 +275,11 @@ struct PureExprScope { /// #10123: `(d, m)` for a body whose first statement is /// `const d = counter % m`. derived: Option<(u32, u32)>, + /// #10199: `(carried, alias)` for a body whose first statement advances a + /// loop-carried index. Both are excluded from bare reads for the same + /// reason `derived` is — inside the clone the real slot is one iteration + /// behind until the trailing commit. + carried: Option<(u32, Option)>, } /// What the walk collects. @@ -311,8 +345,33 @@ fn element_shape_loop_pure_expr_collect( out.props.insert(property.clone()); true } + // #10199: `arr[].prop.length` — a STRING field's JS length. + // Admitted as its own form rather than as "a read of `prop` + // followed by a generic `.length`", because the generic one is a + // property diamond ending in a runtime call, and a call inside this + // clone deletes it. The emitted read tag-tests the loaded word for + // both string representations and side-exits otherwise + // (`expr::element_shape_reads`). + Expr::PropertyGet { .. } if property == "length" => { + element_shape_loop_pure_expr_collect(ctx, object, scope, out) + } _ => false, }, + // #10199: `arr[].prop ? A : B` with constant arms. JS truthiness + // of an arbitrary value is a runtime question the clone is not allowed + // to guess, so the emitted read admits ONLY the two boolean singletons + // and side-exits on everything else. Anything but a tracked element + // read in the condition, or a non-constant arm, declines. + Expr::Conditional { + condition, + then_expr, + else_expr, + } => { + matches!(condition.as_ref(), Expr::PropertyGet { .. }) + && crate::expr::element_shape_reads::integer_arm(then_expr).is_some() + && crate::expr::element_shape_reads::integer_arm(else_expr).is_some() + && element_shape_loop_pure_expr_collect(ctx, condition, scope, out) + } // A bare read of the array, the counter, the element binding or the // derived index as a VALUE could flow it into arbitrary lowering; only // scalar reads the analysis proves numeric are admitted. The element @@ -324,6 +383,10 @@ fn element_shape_loop_pure_expr_collect( Expr::LocalGet(id) => { scope.element_binding != Some(*id) && scope.derived.map(|(d, _)| d) != Some(*id) + // #10199: the carried index and its alias are virtual too — + // their real slots are one iteration behind inside the clone, + // so a bare read would hand out a stale value. + && scope.carried.is_none_or(|(c, alias)| *id != c && alias != Some(*id)) && out.array.is_none_or(|a| a != *id) && (*id == scope.accumulator_id || crate::type_analysis::is_numeric_expr(ctx, expr)) } @@ -381,7 +444,18 @@ fn match_index_form(index: &perry_hir::Expr, scope: &PureExprScope) -> Option None, + // #10199: `arr[c]` and `const index = c; arr[index]` are the same + // subscript; the alias is virtual and carries no obligation of its + // own. + _ => match scope.carried { + Some((carried_id, alias_id)) if carried_id == *id || alias_id == Some(*id) => { + Some(MatchedIndex::Carried { + carried_id, + alias_id, + }) + } + _ => None, + }, }, // `0..=i32::MAX`, so the preheader's `length > k` compare is an i32 // one and the emitted index needs no conversion. @@ -619,41 +693,100 @@ fn anon_shape_field_type_is_compatible( /// Answering `false` for want of the counter's i32 slot cannot happen — the /// matcher requires one for the derived-index form — and would only cost the /// clone, never correctness. -pub(super) fn lower_virtual_clone_binding(ctx: &mut FnCtx<'_>, id: u32) -> bool { - if ctx - .element_shape_loop_facts - .iter() - .any(|fact| fact.element_binding == Some(id)) - { - return true; +pub(super) fn lower_virtual_clone_binding(ctx: &mut FnCtx<'_>, id: u32) -> Result { + enum VirtualBinding { + /// #7771 `const r = arr[j]` and #10199 `const index = c`: pure aliases, + /// no instruction of their own. + Alias, + /// #10123 `const d = j % m`: one `srem i32`. + DerivedMod { + counter_id: u32, + modulus_i32: String, + slot: String, + }, } - let Some((counter_slot, modulus_i32, derived_slot)) = - ctx.element_shape_loop_facts.iter().rev().find_map(|fact| { - let crate::expr::ElementShapeIndex::DerivedMod { + + let Some((scope_id, binding)) = ctx.element_shape_loop_facts.iter().rev().find_map(|fact| { + if fact.element_binding == Some(id) { + return Some((fact.scope_id, VirtualBinding::Alias)); + } + match &fact.index { + crate::expr::ElementShapeIndex::DerivedMod { local_id, modulus_i32, slot, - } = &fact.index - else { - return None; - }; - if *local_id != id { - return None; + } if *local_id == id => Some(( + fact.scope_id, + VirtualBinding::DerivedMod { + counter_id: fact.index_local_id, + modulus_i32: modulus_i32.clone(), + slot: slot.clone(), + }, + )), + crate::expr::ElementShapeIndex::Carried(carried) if carried.alias_id == Some(id) => { + Some((fact.scope_id, VirtualBinding::Alias)) } - Some(( - ctx.i32_counter_slots.get(&fact.index_local_id)?.clone(), - modulus_i32.clone(), - slot.clone(), - )) - }) + _ => None, + } + }) else { + return Ok(false); + }; + + match binding { + VirtualBinding::Alias => {} + VirtualBinding::DerivedMod { + counter_id, + modulus_i32, + slot, + } => { + let Some(counter_slot) = ctx.i32_counter_slots.get(&counter_id).cloned() else { + return Ok(false); + }; + let blk = ctx.block(); + let counter = blk.load(I32, &counter_slot); + let derived = blk.srem(I32, &counter, &modulus_i32); + blk.store(I32, &derived, &slot); + } + } + // #10199: this iteration's index is now in its slot, so this is where the + // shared element deref belongs — for the ONE binding that owns it. + emit_element_prefetch_for(ctx, scope_id, id); + Ok(true) +} + +/// #10199: emit the once-per-iteration element prologue, if `site_local` is the +/// statement the fact designated to own it. +/// +/// Every read in the clone loads the handle this parks, so the designated +/// statement MUST run before any of them — which is why the matcher only ever +/// designates the body's leading virtual statement, and why a fact carrying a +/// prefetch whose index cannot be materialized is a compiler bug rather than a +/// missed optimization (the reads would load an uninitialized alloca). The +/// matcher checks `needs_counter_i32_slot` before the fact is built, so the +/// index is always available here. +pub(super) fn emit_element_prefetch_for(ctx: &mut FnCtx<'_>, scope_id: u32, site_local: u32) { + let Some(fact) = ctx + .element_shape_loop_facts + .iter() + .find(|fact| fact.scope_id == scope_id) + .cloned() else { - return false; + return; + }; + let Some(prefetch) = fact.elem_prefetch.clone() else { + return; }; - let blk = ctx.block(); - let counter = blk.load(I32, &counter_slot); - let derived = blk.srem(I32, &counter, &modulus_i32); - blk.store(I32, &derived, &derived_slot); - true + if prefetch.site_local_id != site_local { + return; + } + let idx_i32 = crate::expr::element_shape_guard::emit_element_shape_index(ctx, &fact) + .expect("a prefetching fact always has a materializable index"); + crate::expr::element_shape_guard::emit_element_shape_prefetch( + ctx, + &fact, + &idx_i32, + &prefetch.handle_slot, + ); } /// Is `array_id` an untyped receiver — the shape-keyed arm's entry condition? @@ -799,7 +932,7 @@ fn match_element_shape_versioned_loop( return None; } - // Store-free body, in one of three admitted shapes (see the module docs): + // Store-free body, in one of four admitted shapes (see the module docs): // // 1. `acc = ].field>` — the original // single statement; @@ -807,89 +940,156 @@ fn match_element_shape_versioned_loop( // element-binding form, the shape real read loops are written in; // 3. `const d = j % m; acc = ` — // #10123's derived-index form, the shape every sequential pass over a - // parsed record array is written in. + // parsed record array is written in; + // 4. #10199's loop-carried form, `c = (a*c + b) % m;` optionally followed + // by `const index = c;`. // - // In 2 and 3 the binding is VIRTUAL inside the fast clone: its `Let` - // emits nothing (form 2) or one `srem i32` (form 3) rather than the - // generic lowering (`stmt/let_stmt.rs`), so the revocation argument (no - // store, no call in the clone) is unchanged. `const`-only, deliberately: a - // `var` binding is function-scoped and observable after the loop, where - // the skipped `Let` would leave the slot holding its pre-loop value. + // In 2, 3 and 4 the leading statement is VIRTUAL inside the fast clone: it + // emits nothing (form 2), one `srem i32` (form 3) or the i64 recurrence + // (form 4) rather than the generic lowering (`stmt/let_stmt.rs`, + // `stmt/element_shape_carried.rs`), so the revocation argument (no store, + // no call in the clone) is unchanged. The bindings are `const`-only, + // deliberately: a `var` binding is function-scoped and observable after the + // loop, where the skipped `Let` would leave the slot holding its pre-loop + // value. Form 4's `c` is the one mutable exception, and it pays for it with + // a write-back (see `element_shape_carried`). // + // #10199 also admits K >= 1 accumulator statements instead of exactly one, + // which is what `sum += rows[i].id; sum += rows[i].name.length; …` needs. // NOTHING else is admitted. let mut element_binding: Option<(u32, u32)> = None; let mut derived: Option<(u32, u32)> = None; - let (acc_id, value) = match body { - [Stmt::Expr(Expr::LocalSet(acc_id, value))] => (acc_id, value), - [Stmt::Let { - id, - mutable: false, - init: Some(binding_init), - .. - }, Stmt::Expr(Expr::LocalSet(acc_id, value))] => { - // The binding must be a plain, loop-owned const local in either - // form. A boxed or captured binding lives in a cell the clone's - // replacement `Let` would leave stale for an observer outside the - // clone; a module-global id is not a body-scoped binding at all. - if *id == counter_id - || ctx.boxed_vars.contains(id) - || ctx.module_globals.contains_key(id) - || ctx.closure_captures.contains_key(id) - { - return None; + let mut carried: Option = None; + let mut carried_alias: Option = None; + let mut rest: &[Stmt] = body; + + // (4) The recurrence, which must be the body's FIRST statement: the index + // it produces is read by every statement after it, and a read BEFORE it + // would see a value the preheader bounded but the commit protocol does not + // describe. + if let Some(first) = rest.first() { + if let Some(matched) = super::element_shape_carried::match_carried_update( + ctx, first, counter_id, condition?, update, body, + ) { + carried = Some(matched); + rest = &rest[1..]; + } + } + + // (2)/(3)/(4-alias) The leading virtual binding. + if let Some(Stmt::Let { + id, + mutable: false, + init: Some(binding_init), + .. + }) = rest.first() + { + // The binding must be a plain, loop-owned const local in every form. A + // boxed or captured binding lives in a cell the clone's replacement + // `Let` would leave stale for an observer outside the clone; a + // module-global id is not a body-scoped binding at all. + if *id == counter_id + || ctx.boxed_vars.contains(id) + || ctx.module_globals.contains_key(id) + || ctx.closure_captures.contains_key(id) + { + return None; + } + match binding_init { + Expr::IndexGet { object, index } if carried.is_none() => { + let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = + (object.as_ref(), index.as_ref()) + else { + return None; + }; + // Same receiver/index discipline as the walk's IndexGet + // arm: the fetch must be `arr[counter]` exactly. + if *idx_id != counter_id || *arr_id == counter_id || *id == *arr_id { + return None; + } + element_binding = Some((*id, *arr_id)); } - match binding_init { - Expr::IndexGet { object, index } => { - let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = - (object.as_ref(), index.as_ref()) - else { - return None; - }; - // Same receiver/index discipline as the walk's IndexGet - // arm: the fetch must be `arr[counter]` exactly. - if *idx_id != counter_id || *arr_id == counter_id || *id == *arr_id { - return None; - } - element_binding = Some((*id, *arr_id)); + // #10123. `%` on two locals, the left one the counter. The + // modulus is validated below (it must be readable and + // loop-invariant), and the RANGE obligation — `1 <= m` so the + // `srem` cannot divide by zero, `m <= length` so every derived + // index is in bounds — is discharged in the preheader, which + // is also where a non-number / fractional `m` sends the loop + // to the slow clone. + Expr::Binary { + op: BinaryOp::Mod, + left, + right, + } if carried.is_none() => { + let (Expr::LocalGet(num_id), Expr::LocalGet(modulus_id)) = + (left.as_ref(), right.as_ref()) + else { + return None; + }; + if *num_id != counter_id || *modulus_id == counter_id || *id == *modulus_id { + return None; } - // #10123. `%` on two locals, the left one the counter. The - // modulus is validated below (it must be readable and - // loop-invariant), and the RANGE obligation — `1 <= m` so the - // `srem` cannot divide by zero, `m <= length` so every derived - // index is in bounds — is discharged in the preheader, which - // is also where a non-number / fractional `m` sends the loop - // to the slow clone. - Expr::Binary { - op: BinaryOp::Mod, - left, - right, - } => { - let (Expr::LocalGet(num_id), Expr::LocalGet(modulus_id)) = - (left.as_ref(), right.as_ref()) - else { - return None; - }; - if *num_id != counter_id || *modulus_id == counter_id || *id == *modulus_id { - return None; - } - if !modulus_local_is_admissible(ctx, *modulus_id, condition?, update, body) { - return None; - } - derived = Some((*id, *modulus_id)); + if !modulus_local_is_admissible(ctx, *modulus_id, condition?, update, body) { + return None; } - _ => return None, + derived = Some((*id, *modulus_id)); } - (acc_id, value) + // #10199: `const index = c` after the recurrence. A pure alias, so + // it emits nothing at all — the subscript resolves to the carried + // slot either way. + Expr::LocalGet(aliased) => { + let matched = carried.as_ref()?; + if *aliased != matched.carried_id || *id == matched.modulus_id { + return None; + } + carried_alias = Some(*id); + } + _ => return None, } - _ => return None, - }; + rest = &rest[1..]; + } + + // (1) One or more accumulator statements over the SAME local. + let mut acc_id: Option<&u32> = None; + let mut values: Vec<&perry_hir::Expr> = Vec::new(); + for stmt in rest { + let Stmt::Expr(Expr::LocalSet(id, value)) = stmt else { + return None; + }; + match acc_id { + None => acc_id = Some(id), + Some(seen) if seen == id => {} + Some(_) => return None, + } + values.push(value.as_ref()); + } + let acc_id = acc_id?; if *acc_id == counter_id || !ctx.locals.contains_key(acc_id) || ctx.boxed_vars.contains(acc_id) || ctx.module_globals.contains_key(acc_id) + || carried.is_some_and(|c| c.carried_id == *acc_id || c.modulus_id == *acc_id) { return None; } + // #10199: statements 2..K are folded into statement 1 for the fast clone + // (`acc = a; acc = acc + b` ≡ `acc = (a) + b`), so that the WHOLE iteration + // commits once, after every side exit it can take. Each of them must + // therefore be `acc ` with `acc` as the left operand — the shape + // every compound assignment lowers to — and the right operand must not read + // `acc`, or substituting it would make it see the pre-statement value. + for value in &values[1..] { + let Expr::Binary { left, right, .. } = value else { + return None; + }; + if !matches!(left.as_ref(), Expr::LocalGet(id) if id == acc_id) { + return None; + } + if super::element_shape_carried::expr_reads_local(right, *acc_id) { + return None; + } + } + // The binding form pins the array before the walk runs, so a body mixing // `r.field` with `other[j].field` is declined by the walk's one-array rule. let scope = PureExprScope { @@ -897,15 +1097,22 @@ fn match_element_shape_versioned_loop( accumulator_id: *acc_id, element_binding: element_binding.map(|(id, _)| id), derived, + carried: carried.map(|c| (c.carried_id, carried_alias)), }; - if scope.element_binding == Some(*acc_id) || derived.map(|(d, _)| d) == Some(*acc_id) { + if scope.element_binding == Some(*acc_id) + || derived.map(|(d, _)| d) == Some(*acc_id) + || carried_alias == Some(*acc_id) + { return None; } let mut facts = PureExprFacts { array: element_binding.map(|(_, arr_id)| arr_id), ..PureExprFacts::default() }; - if !element_shape_loop_pure_expr_collect(ctx, value, &scope, &mut facts) { + if !values + .iter() + .all(|value| element_shape_loop_pure_expr_collect(ctx, value, &scope, &mut facts)) + { return None; } let array_id = facts.array?; @@ -927,12 +1134,35 @@ fn match_element_shape_versioned_loop( return None; } } + // #10199: a matched recurrence whose value nothing subscripts with would + // still have its `LocalSet` lowered generically — a `frem` libcall inside + // the clone, which deletes it. Decline instead, so the loop keeps whatever + // lowering it has today. + match (carried.as_ref(), index) { + (Some(matched), MatchedIndex::Carried { .. }) => { + if matched.carried_id == array_id + || matched.modulus_id == array_id + || carried_alias == Some(array_id) + { + return None; + } + } + // A recurrence the subscripts do not use, or a carried subscript with + // no recurrence (which the walk's scope makes unreachable). + (Some(_), _) | (None, MatchedIndex::Carried { .. }) => return None, + (None, _) => {} + } match bound { ElementShapeLoopBound::Local(bound_id) => { if bound_id == array_id || bound_id == *acc_id || Some(bound_id) == scope.element_binding || derived.map(|(d, _)| d) == Some(bound_id) + // The bound is materialized ONCE in the preheader, so a bound + // the recurrence rewrites every iteration would be stale. The + // carried local is also not readable bare inside the clone. + || carried.is_some_and(|c| c.carried_id == bound_id) + || carried_alias == Some(bound_id) { return None; } @@ -975,12 +1205,6 @@ fn match_element_shape_versioned_loop( return None; } - for prop in &facts.props { - if CLASS_FIELD_LOOP_PROP_DENYLIST.contains(&prop.as_str()) { - return None; - } - } - let identity = match element_class_name(ctx, array_id, counter_id) { Some(class_name) => { // The class arm keeps its ORIGINAL grammar. Its bounds argument is @@ -999,6 +1223,42 @@ fn match_element_shape_versioned_loop( None => return None, }; + // The property denylist, which is arm-specific (#10199). + // + // The CLASS arm's list exists because its read bakes in a compile-time + // packed slot index while the surrounding lowering may route the name + // somewhere else entirely (`length` header loads, `errors` runtime calls, + // accessor-ish names) — a name collision there costs the call-free + // guarantee. + // + // The SHAPE arm bakes in nothing: the preheader asks the runtime for the + // inline slot of that exact key in that exact ordinary ShapeId and declines + // on `-1`, and the per-element residual pins `obj_type == GC_TYPE_OBJECT` + // with no per-object descriptors. Every receiver kind whose builtin branch + // could answer a name differently — a function's `name`, an array's + // `length`, a Map's `size`, an AggregateError's `errors` — fails that + // residual, and for a plain record an own data property SHADOWS the + // prototype name it collides with, which is exactly what the inline slot + // holds. The whole list would otherwise have cost the benchmark its + // `fields` shape outright, for a field literally called `name`. + // + // `__proto__` stays denied, and not because of JavaScript: node gives + // `JSON.parse('{"__proto__":1}')` an OWN `__proto__` data property and + // reads `1` back from it, so the inline slot would be right. It is denied + // because Perry's generic property path may special-case the name ahead of + // own-property lookup, and the clone must agree with the clone-of, not with + // the spec, wherever those two differ. + const SHAPE_PROP_DENYLIST: &[&str] = &["__proto__"]; + let denylist = match identity { + ElementShapeIdentity::Class { .. } => CLASS_FIELD_LOOP_PROP_DENYLIST, + ElementShapeIdentity::Shape => SHAPE_PROP_DENYLIST, + }; + for prop in &facts.props { + if denylist.contains(&prop.as_str()) { + return None; + } + } + // The declared accumulator type is only a candidate: the lowering // validates the accumulator's current NaN-box tag in the preheader before // installing the numeric fact for the fast clone. The class arm keeps its @@ -1018,6 +1278,40 @@ fn match_element_shape_versioned_loop( return None; } + // #10199: the fast clone's body, when the two rewrites apply. `lead` is + // whatever statements the walk consumed ahead of the accumulator run (the + // recurrence and/or the virtual binding), kept verbatim. + let lead = &body[..body.len() - rest.len()]; + let fast_body = (values.len() > 1 || carried.is_some()).then(|| { + let mut stmts: Vec = lead.to_vec(); + // `acc = v1; acc = acc p2; acc = acc p3` + // ≡ `acc = ((v1) p2) p3` + // — the same operations on the same operands in the same order, which + // is what keeps the folded float arithmetic bit-identical. + let mut folded = values[0].clone(); + for value in &values[1..] { + let Expr::Binary { op, right, .. } = value else { + unreachable!("the accumulator fold validated every statement's shape"); + }; + folded = Expr::Binary { + op: *op, + left: Box::new(folded), + right: right.clone(), + }; + } + stmts.push(Stmt::Expr(Expr::LocalSet(*acc_id, Box::new(folded)))); + if let Some(matched) = carried.as_ref() { + // The commit marker (`stmt/element_shape_carried.rs`). It is the + // LAST statement, so it runs only once the iteration is past every + // exit it could take. + stmts.push(Stmt::Expr(Expr::LocalSet( + matched.carried_id, + Box::new(Expr::LocalGet(matched.carried_id)), + ))); + } + stmts + }); + Some(ElementShapeVersionedLoop { counter_id, bound, @@ -1027,6 +1321,8 @@ fn match_element_shape_versioned_loop( index, element_binding: scope.element_binding, accumulator_id: *acc_id, + carried, + fast_body, }) } @@ -1200,10 +1496,25 @@ pub(super) fn lower_element_shape_versioned_for( // floor of 1 — `srem` by zero is undefined behaviour, and `x % 0` is NaN // in JS, so a zero modulus is a slow-clone case rather than something the // clone may compute. - let modulus_i32: Option = match matched.index { - MatchedIndex::DerivedMod { modulus_id, .. } => Some(materialize_loop_i32( + // + // #10199's carried recurrence takes the same obligation for the same + // reason, and one more of its own: its ENTRY value must be a non-negative + // integral i32, because the whole i64 recurrence bound + // (`a * i32::MAX + b < 2^53`, and a non-negative dividend for `srem` to + // agree with JS `%`) is stated over that range. A fractional, negative or + // non-number `cursor` at loop entry is a slow-clone case. + let modulus_i32: Option = match (&matched.index, &matched.carried) { + (MatchedIndex::DerivedMod { modulus_id, .. }, _) => Some(materialize_loop_i32( + ctx, + *modulus_id, + 1, + i32::MAX, + &slow_pre_label, + "element_shape.loop.modulus", + )?), + (MatchedIndex::Carried { .. }, Some(carried)) => Some(materialize_loop_i32( ctx, - modulus_id, + carried.modulus_id, 1, i32::MAX, &slow_pre_label, @@ -1211,6 +1522,25 @@ pub(super) fn lower_element_shape_versioned_for( )?), _ => None, }; + let carried_slot: Option = match &matched.carried { + Some(carried) => { + let entry = materialize_loop_i32( + ctx, + carried.carried_id, + 0, + i32::MAX, + &slow_pre_label, + "element_shape.loop.carried", + )?; + // Entry-block alloca: LLVM lowers a non-entry `alloca` as a real + // stack bump with no restore, and this one is written every + // iteration. + let slot = ctx.func.alloca_entry(I32); + ctx.block().store(I32, &entry, &slot); + Some(slot) + } + None => None, + }; let trip_count = match &materialized_bound { Some(bound) => { @@ -1225,11 +1555,14 @@ pub(super) fn lower_element_shape_versioned_for( (MatchedIndex::Constant(k), _) => { crate::expr::element_shape_guard::ElementShapeIndexBound::Constant(*k) } - (MatchedIndex::DerivedMod { .. }, Some(modulus)) => { + // #10199: the recurrence's result is `srem` of a non-negative dividend + // by `m`, so it lands in `[0, m)` exactly like the derived index and + // takes exactly the same preheader obligation, `m <= length`. + (MatchedIndex::DerivedMod { .. } | MatchedIndex::Carried { .. }, Some(modulus)) => { crate::expr::element_shape_guard::ElementShapeIndexBound::Modulus(modulus.as_str()) } - (MatchedIndex::DerivedMod { .. }, None) => { - unreachable!("a derived index always materializes its modulus") + (MatchedIndex::DerivedMod { .. } | MatchedIndex::Carried { .. }, None) => { + unreachable!("a modulo-derived index always materializes its modulus") } }; let expected_class_id_str = match &matched.identity { @@ -1323,8 +1656,58 @@ pub(super) fn lower_element_shape_versioned_for( // iteration. slot: ctx.func.alloca_entry(I32), }, + MatchedIndex::Carried { + carried_id, + alias_id, + } => { + let carried = matched + .carried + .as_ref() + .expect("a carried index always matches a recurrence"); + crate::expr::ElementShapeIndex::Carried(Box::new(crate::expr::CarriedIndex { + local_id: carried_id, + alias_id, + coeff_a: carried.coeff_a, + coeff_b: carried.coeff_b, + modulus_i32: modulus_i32 + .clone() + .expect("a carried index always materializes its modulus"), + slot: carried_slot + .clone() + .expect("a carried index always materializes its entry value"), + commit_slot: ctx + .locals + .get(&carried_id) + .cloned() + .expect("the matcher required a directly-addressable local"), + })) + } }; + // #10199: the shared once-per-iteration element deref, hung off the body's + // leading virtual statement. Shape-keyed only: the class-keyed arm's reads + // are raw doubles behind a `GC_OBJ_TYPED_LAYOUT_INTACT` residual whose + // emitted IR #7480's census pins, and nothing in #10199 measures it. + let elem_prefetch = shape_keyed + .then(|| { + let site_local_id = match (&fact_index, matched.element_binding) { + (_, Some(binding)) => Some(binding), + (crate::expr::ElementShapeIndex::DerivedMod { local_id, .. }, _) => Some(*local_id), + // With an alias the prologue belongs to the alias `Let`, which + // runs after the recurrence; without one it belongs to the + // recurrence itself. Exactly one statement owns it either way. + (crate::expr::ElementShapeIndex::Carried(carried), _) => { + Some(carried.alias_id.unwrap_or(carried.local_id)) + } + _ => None, + }?; + Some(crate::expr::ElementPrefetch { + site_local_id, + handle_slot: ctx.func.alloca_entry(I64), + }) + }) + .flatten(); + let scope_id = ctx.next_loop_proof_scope_id(); let fast_scan_start = ctx.func.num_blocks(); ctx.current_block = fast_pre_idx; @@ -1333,6 +1716,7 @@ pub(super) fn lower_element_shape_versioned_for( array_local_id: matched.array_id, index_local_id: matched.counter_id, index: fact_index, + elem_prefetch, shape_keyed, scope_id, class_name: report_class, @@ -1341,6 +1725,7 @@ pub(super) fn lower_element_shape_versioned_for( side_exit_label: slow_pre_label.clone(), statically_layout_proven, fields, + synthesized_body: matched.fast_body.is_some(), element_binding: matched.element_binding, numeric_accumulator: matched.accumulator_id, }); @@ -1349,7 +1734,7 @@ pub(super) fn lower_element_shape_versioned_for( init, condition, update, - body, + matched.fast_body.as_deref().unwrap_or(body), "for.element_shape_fast", Some((matched.counter_id, guard.bound_i32)), ); diff --git a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs index 08e96be315..fcbaa46dfb 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs @@ -402,14 +402,34 @@ fn assert_fast_clone_is_entered(ir: &str) { fn fast_clone_slice(ir: &str) -> String { let mut owned = String::new(); let mut in_fast_block = false; + // #10199: the emitted text carries the function TWICE (same block labels), + // so every block would be collected twice and any `matches().count()` + // assertion against the slice would read double. Stop at the first repeated + // label, which is where the second copy begins — `contains` assertions are + // unaffected, and counting one iteration of the clone is what makes "the + // residual check happens ONCE" expressible at all. + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); for line in ir.split_inclusive('\n') { let trimmed = line.trim_end(); // A block DEFINITION starts at column 0 and ends in `:`; anything else // belongs to whichever block was last opened. if !line.starts_with(char::is_whitespace) && trimmed.ends_with(':') { + // #10199 added three more: the string-`.length` decode + // (`element_shape.strlen*`), the boolean ternary's admitted arm + // (`element_shape.bool`), and nothing for the shared prefetch, + // which reuses `element_shape.load`. Every block the clone can + // execute must be listed, or the negatives below go vacuous for it. in_fast_block = trimmed.starts_with("for.element_shape_fast.") || trimmed.starts_with("element_shape.load") - || trimmed.starts_with("element_shape.number"); + || trimmed.starts_with("element_shape.number") + || trimmed.starts_with("element_shape.strlen") + || trimmed.starts_with("element_shape.bool"); + // Only a repeated CLONE label means the second copy: unrelated + // functions share ordinary labels (`entry:`), and breaking on one + // of those would slice away the clone entirely. + if in_fast_block && !seen.insert(trimmed.to_string()) { + break; + } } if in_fast_block { owned.push_str(line); diff --git a/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs b/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs index 94ade02cc7..fe47ad0e89 100644 --- a/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs @@ -450,21 +450,78 @@ fn an_untracked_local_index_declines() { } #[test] -fn a_denylisted_property_declines_the_shape_keyed_arm() { - // `length`, `name`, `constructor`, … are answered by the runtime or by the - // prototype, not out of an inline slot, so a shape's key position for one - // would be the wrong answer even when it exists. +fn the_shape_keyed_arm_denies_only_proto() { + // #10199 narrowed this list. The CLASS arm still denies every name with a + // dedicated branch in the property dispatch, because its read bakes in a + // compile-time packed slot. The SHAPE arm bakes in nothing: the preheader + // asks the runtime for that exact key's inline slot in that exact ordinary + // ShapeId and declines on `-1`, and the residual pins + // `obj_type == GC_TYPE_OBJECT` with no per-object descriptors — so every + // receiver whose builtin branch could answer a name differently (a + // function's `name`, an array's `length`, a Map's `size`) is already + // excluded, and for a plain record an own data property shadows the + // prototype name it collides with. Denying the whole list cost the access + // benchmark its `fields` shape outright, for a field called `name`. + for property in ["length", "name", "constructor", "size", "message"] { + let ir = emit(&untyped_param_module( + Type::Any, + Type::Number, + vec![untyped_accumulate(untyped_elem_field( + Expr::LocalGet(U_COUNTER_ID), + property, + ))], + )); + assert!( + ir.contains("element_shape.loop.fast.preheader"), + "`{property}` is an ordinary own inline slot on a parsed record; \ + the shape-keyed arm must serve it" + ); + } + // `__proto__` stays denied — not because of JavaScript (node gives + // `JSON.parse('{\"__proto__\":1}')` an own data property and reads `1` + // back), but because Perry's generic property path may special-case the + // name ahead of own-property lookup, and the clone must agree with the + // path it is a clone of. let ir = emit(&untyped_param_module( Type::Any, Type::Number, vec![untyped_accumulate(untyped_elem_field( Expr::LocalGet(U_COUNTER_ID), - "length", + "__proto__", + ))], + )); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "`__proto__` must decline the shape-keyed clone" + ); +} + +#[test] +fn the_class_keyed_arm_keeps_the_full_property_denylist() { + // The narrowing above is shape-arm-only: a class-keyed read bakes in a + // packed slot index while the surrounding lowering may route the name + // somewhere else entirely, which is what the list has always protected. + let ir = emit(&element_shape_module( + vec![Stmt::Expr(Expr::LocalSet( + SUM_ID, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(SUM_ID)), + right: Box::new(Expr::PropertyGet { + object: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ARRAY_ID)), + index: Box::new(Expr::LocalGet(COUNTER_ID)), + }), + property: "length".to_string(), + byte_offset: 0, + }), + }), ))], + None, )); assert!( !ir.contains("element_shape.loop.fast.preheader"), - "a denylisted property must decline the clone" + "a denylisted property must still decline the CLASS-keyed clone" ); } @@ -489,3 +546,9 @@ fn a_declared_but_unresolvable_element_type_still_declines() { "an unresolvable declared element type must decline both arms" ); } + +/// #10199's `fields` and `random` shapes — a child module so it inherits both +/// this file's shape-keyed assertions and `element_shape_loop_tests`'s slicing +/// helpers. +#[path = "element_shape_fields_random_tests.rs"] +mod fields_random; diff --git a/crates/perry-codegen/src/stmt/let_stmt.rs b/crates/perry-codegen/src/stmt/let_stmt.rs index 28ec377674..288e799086 100644 --- a/crates/perry-codegen/src/stmt/let_stmt.rs +++ b/crates/perry-codegen/src/stmt/let_stmt.rs @@ -31,7 +31,7 @@ pub(crate) fn lower_let( // binding is VIRTUAL — the element binding emits nothing and the derived // index emits one `srem`. Both live with the clone, which is where their // soundness arguments and the preheader that validated them are. - if super::element_shape_loop::lower_virtual_clone_binding(ctx, id) { + if super::element_shape_loop::lower_virtual_clone_binding(ctx, id)? { return Ok(()); } // `let C = SomeClass` aliases the local `C` to the class diff --git a/crates/perry-codegen/src/stmt/mod.rs b/crates/perry-codegen/src/stmt/mod.rs index cc407d30c0..5b9ac4f33e 100644 --- a/crates/perry-codegen/src/stmt/mod.rs +++ b/crates/perry-codegen/src/stmt/mod.rs @@ -20,6 +20,7 @@ mod cached_field_index_return; #[cfg(test)] mod class_field_loop_tests; mod counter_range; +mod element_shape_carried; mod element_shape_loop; #[cfg(test)] mod element_shape_loop_tests; @@ -281,6 +282,15 @@ fn lower_return_expr(ctx: &mut FnCtx<'_>, expr: &perry_hir::Expr) -> Result, stmt: &Stmt) -> Result<()> { match stmt { Stmt::Expr(e) => { + // #10199: the element-shape fast clone's carried-index statements + // (the recurrence and its trailing write-back) are lowered + // VIRTUALLY, exactly like the `Let` bindings in `let_stmt.rs` — + // the generic lowering of `c = (c * 17 + 7) % length` is an + // `frem` libcall, and a call inside the clone DELETES it (#7690). + // Outside a clone the fact vector is empty and this is a no-op. + if element_shape_carried::lower_virtual_carried_stmt(ctx, e)? { + return Ok(()); + } let prev_discard = ctx.discard_expr_value; ctx.discard_expr_value = true; // #7590: the non-leaking companion. `lower_expr` takes this at the diff --git a/crates/perry-codegen/src/type_analysis/numeric.rs b/crates/perry-codegen/src/type_analysis/numeric.rs index e3677034e2..e3c20605f2 100644 --- a/crates/perry-codegen/src/type_analysis/numeric.rs +++ b/crates/perry-codegen/src/type_analysis/numeric.rs @@ -356,6 +356,18 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { Expr::Logical { left, right, .. } => { is_numeric_expr(ctx, left) && is_numeric_expr(ctx, right) } + // #10199: `arr[i]. ? 1 : 0` inside an element-shape fast + // clone. Deliberately NOT the general "both arms are numeric" rule — a + // general ternary's CONDITION is an arbitrary JS truthiness test, which + // is a runtime call; this arm is true only for the exact shape the + // clone lowers itself, where the condition is a tracked element read + // admitted as one of the two boolean singletons and everything else + // side-exits. + Expr::Conditional { .. } + if crate::expr::element_shape_reads::cloned_bool_select_read(ctx, e).is_some() => + { + true + } // `obj.field` where the field is declared as `number` on the // owning class. Without this, `this.value + 1` in a hot loop // wraps the field load in `js_number_coerce` which prevents @@ -374,6 +386,17 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { if property == "length" && expression_has_numeric_length(ctx, object) { return true; } + // #10199: `arr[i]..length` inside an element-shape + // fast clone. The receiver is an untyped element read, so the + // declared-type answer above cannot see a string there; the clone's + // own lowering tag-tests the loaded word for both string + // representations and side-exits when it is not one, which is a + // stronger proof than an annotation. Without this the `+` consuming + // it bails to `js_dynamic_string_or_number_add` and the call + // deletes the clone. + if crate::expr::element_shape_reads::cloned_string_length_read(ctx, e).is_some() { + return true; + } // repsel #7480 step 3: inside an element-shape fast clone a tracked // `arr[i].field` read is a GUARD-PROVEN raw double. The class-keyed // arm gets that from the residual check's From 5e73353ea272c49ef30114f8d3d4316d21f6aa48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 12:50:11 +0200 Subject: [PATCH 06/10] test(gap): cover the carried index, the accumulator fold and the two new reads 40 cases in test_gap_json_record_loop_clone.ts, all byte-identical to node 26.5.1: the recurrence with and without its alias, the carried value read after the loop, negative and fractional entry values, a zero and an over-long modulus, a multiplier past the exact-double range, a mid-loop side exit whose sum AND final cursor observe the write-back protocol, SSO and heap names, a non-string and a null name, five non-boolean `active` values, a two-statement fold whose second read side-exits, and records carrying own `name`/`length`/ `size` properties. Plus the changelog fragment with the measured 12-cell table and the instruction counts. (cherry picked from commit bf0904a9385a3e6f0b6110f926e783df2916f6f5) --- .../10185-element-shape-fields-random.md | 157 +++++++++ crates/perry-codegen/src/expr/binary.rs | 2 +- .../src/expr/element_shape_guard.rs | 6 +- .../src/expr/element_shape_reads.rs | 2 +- crates/perry-codegen/src/expr/mod.rs | 14 +- crates/perry-codegen/src/expr/shadow_slot.rs | 4 +- .../src/stmt/element_shape_carried.rs | 2 +- .../stmt/element_shape_fields_random_tests.rs | 6 +- .../src/stmt/element_shape_loop.rs | 44 +-- .../src/stmt/element_shape_loop_tests.rs | 4 +- .../stmt/element_shape_shape_keyed_tests.rs | 4 +- crates/perry-codegen/src/stmt/mod.rs | 2 +- .../src/type_analysis/numeric.rs | 4 +- test-files/test_gap_json_record_loop_clone.ts | 298 ++++++++++++++++++ 14 files changed, 502 insertions(+), 47 deletions(-) create mode 100644 changelog.d/10185-element-shape-fields-random.md diff --git a/changelog.d/10185-element-shape-fields-random.md b/changelog.d/10185-element-shape-fields-random.md new file mode 100644 index 0000000000..788c133fdf --- /dev/null +++ b/changelog.d/10185-element-shape-fields-random.md @@ -0,0 +1,157 @@ +Serve the JSON access benchmark's `fields` and `random` shapes from the +element-shape versioned loop clone (#7480 / #5093 / #10171). Stacked on +#10171, which made the clone fire for `JSON.parse`'d record arrays at all. + +#10171 took the benchmark's `repeat` and `sequential` modes to parity. The two +remaining modes got nothing, for three separate reasons — and because a loop is +admitted as a whole, any one of them cost the entire clone: + +```ts +// random: a loop-carried index +cursor = (cursor * 17 + 7) % length; +const index = cursor; +sum += rows[index].id; + +// fields: three accumulator statements, one string read, one boolean ternary +const index = i % length; +sum += rows[index].id; +sum += rows[index].name.length; +sum += rows[index].active ? 1 : 0; +``` + +* **A loop-carried index** (`stmt/element_shape_carried.rs`). The recurrence is + folded to one affine pair `(a, b)` and evaluated as `srem i64` — the generic + `%` is an `frem`, which on aarch64 is a libm call, and a call inside this + clone does not slow it down, it DELETES it (#7690). Three obligations, all + discharged once in the preheader: the modulus materializes as an i32 in + `1..=i32::MAX` (so `m <= length` puts every index in bounds with no per-read + test), the carried binding's ENTRY value materializes as a non-negative + integral i32, and `a`/`b` are required non-negative so the dividend cannot go + negative — JS `%` returns a *negative* remainder for a negative dividend, + which is an out-of-bounds subscript rather than a slow path. + + The matcher also tracks the largest magnitude any sub-expression of the + recurrence can reach and declines above 2^53. "Fits i64" is necessary and NOT + sufficient: JavaScript evaluates the recurrence in doubles, and + `cursor * 1e9 + 1` fits i64 comfortably while losing its low bits in the f64 + the program actually runs. The bound is tracked per NODE rather than taken + from the folded pair, because folding cancels: `c * 1000 - c * 999` is + `1 * c`, but JavaScript still evaluates both thousand-fold products. + +* **Where the carried write-back goes is a correctness question, not a + scheduling one.** The residual side exit resumes the CURRENT iteration in the + slow clone, which re-runs the whole body — the recurrence included. So the + commit to the real binding is the LAST statement of the iteration: a + mid-iteration exit then leaves the binding holding that iteration's entry + value and the slow clone advances it exactly once. A write-back at the update + site double-applies it, and every later index is silently a different — still + in-bounds, still a valid record — one. + +* **K accumulator statements**, folded into one for the fast clone + (`acc = a; acc = acc + b` ≡ `acc = (a) + b`: the same operations on the same + operands in the same order, so the float result is bit-identical). Same + reason as above — the whole iteration must commit once, past every exit it + can take, or a tag test that fails on the third read leaves the first two + already applied when the slow clone re-runs it. Statements 2..K must read the + accumulator as the LEFT operand and must not read it in the addend, or the + substitution would change which value the addend sees. + +* **`arr[i].prop.length` and `arr[i].prop ? A : B`** + (`expr/element_shape_reads.rs`). The preheader proves which inline slot holds + a property; it proves nothing about what is in it. So each read tag-tests the + loaded word and side-exits when the answer is not the representation it is + about to assume: both string forms for `.length` (heap `utf16_len`, and the + SSO length byte — the same decode the runtime's own `.length` arms use), and + the two boolean singletons by exact NaN-box bit pattern for the ternary. + JS truthiness of `0` / `""` / `null` / an object is a runtime question the + clone does not guess, and a non-constant ternary arm is not admitted at all. + +* **One residual check per iteration, not one per read.** Three reads of + `rows[index]` were three element loads and three header/ShapeId checks. The + body's leading virtual binding now emits the deref and the residual once and + parks the masked handle in an entry alloca. That is also what makes the + multi-read side exit correct: every residual exit now precedes every store. + +* **The property denylist is arm-specific now.** The class-keyed arm keeps the + full list — its read bakes in a compile-time packed slot index while the + surrounding lowering may route the name elsewhere. The shape-keyed arm bakes + in nothing: it asks the runtime for that exact key's inline slot in that exact + ordinary ShapeId and declines on `-1`, and the residual pins + `obj_type == GC_TYPE_OBJECT` with no per-object descriptors — so every + receiver whose builtin branch could answer a name differently (a function's + `name`, an array's `length`, a Map's `size`) is already excluded, and for a + plain record an own data property SHADOWS the prototype name it collides + with. Only `__proto__` stays denied, and not because of JavaScript (node + gives `JSON.parse('{"__proto__":1}')` an own data property and reads `1` back + from it): Perry's generic property path may special-case the name ahead of + own-property lookup, and the clone must agree with the path it is a clone of + wherever the two differ. The full list would otherwise have cost the + benchmark its `fields` mode outright, for a field literally called `name`. + +**Measured** (ns/iter, 1M iterations, 5 interleaved rounds, best of 5, +`PERRY_NO_AUTO_OPTIMIZE=1`, on a loaded shared host; `new` and `base` are both +built from this worktree, `base` at #10171's head `2b77e7d4fe`): + +| cell | base (#10171) | new | node 26.5.1 | bun 1.3.14 | new / best | +|---|---:|---:|---:|---:|---:| +| 16k `repeat` | 4.15 | 4.19 | 3.24 | 5.03 | 1.293 | +| 16k `sequential` | 4.21 | 4.17 | 4.35 | 6.14 | 0.959 | +| 16k `random` | 15.38 | **4.99** | 7.10 | 9.61 | **0.703** | +| 16k `fields` | 25.76 | **6.19** | 5.75 | 8.35 | **1.077** | +| 1m `repeat` | 4.12 | 4.19 | 3.57 | 5.25 | 1.174 | +| 1m `sequential` | 4.24 | 4.18 | 9.35 | 8.06 | 0.519 | +| 1m `random` | 15.90 | **5.25** | 12.66 | 11.66 | **0.450** | +| 1m `fields` | 29.02 | **6.31** | 12.68 | 15.41 | **0.498** | +| 20m `repeat` | 4.19 | 4.17 | 3.52 | 5.13 | 1.185 | +| 20m `sequential` | 4.29 | 4.37 | 7.48 | 8.83 | 0.584 | +| 20m `random` | 27.56 | **5.60** | 9.45 | 10.64 | **0.593** | +| 20m `fields` | 27.26 | **6.22** | 11.53 | 14.93 | **0.539** | + +The two targeted shapes move 3.1x-4.9x and land below the better of node and +bun on five of their six cells. The sixth, 16k `fields`, is the one that does +not reach parity: 6.08 against node's 5.83 on a 12-round re-measure (1.043x). +That is the smallest fixture, where the whole array is in L1 and the guard +overhead is the only thing left — node's own `fields` is just 1.34x its +`sequential` there, because its inline caches have type feedback proving `name` +is a string and `active` a boolean, while this clone tag-tests both on every +read. Paying that test is what lets it side-exit instead of deoptimize, so it +is a floor of the design rather than a missing peephole. Reported rather than +rounded off. + +`repeat` and `sequential`, which #10171 already served, are unchanged within +noise in both wall clock and instruction count — `sequential`'s single read has +nothing to share, and `repeat`'s constant index has no leading binding to hang +the prologue off. + +Instructions retired per iteration on the 16k fixture (2M iterations minus a +0-iteration run, `/usr/bin/time -l`): + +| shape | base | new | +|---|---:|---:| +| random | 207.9 | 29.1 | +| fields | 466.6 | 47.0 | +| sequential | 25.1 | 25.1 | +| repeat | 16.1 | 15.8 | + +`random` lands one recurrence above `sequential`'s 25, and `fields` two extra +guarded reads plus a string decode and a select above it — which is what the +shared residual buys, and the shape the numbers have to have if the clone is +really being entered rather than merely emitted (#10171's own lesson: a clone +can be `cond_br`-entered and never executed while every IR-census assertion +passes, and only the wall clock notices). + +Validation: `test-files/test_gap_json_record_loop_clone.ts` gains 40 cases — +the recurrence with and without its alias, the carried value read after the +loop, negative and fractional entry values, a zero and an over-long modulus, a +multiplier past the exact-double range, a mid-loop side exit whose sum AND +final cursor both observe the write-back protocol, SSO and heap `name`s, a +non-string and a null `name`, five non-boolean `active` values, a two-statement +fold whose second read side-exits, and records with own `name`/`length`/`size` +properties — all byte-identical to `node --experimental-strip-types` 26.5.1. +70 codegen IR-census tests (`element_shape_fields_random_tests.rs` and its +siblings), every positive paired with a sabotage case asserting the clone is +ABSENT. Seeded GC stress on the gap binary +(`PERRY_GC_SCHEDULE_SEED=1..4 PERRY_GC_SCHEDULE_RATE=0.2 +PERRY_GC_PROTECT_FROMSPACE=1 PERRY_GC_PROTECT_FROMSPACE_DEPTH=32`): output +identical across all four seeds and to node, with 3–11 copying minors and +~14.4k moved objects per run, so the instrument's subject was live. diff --git a/crates/perry-codegen/src/expr/binary.rs b/crates/perry-codegen/src/expr/binary.rs index ce24554dc7..ebeac06c0f 100644 --- a/crates/perry-codegen/src/expr/binary.rs +++ b/crates/perry-codegen/src/expr/binary.rs @@ -504,7 +504,7 @@ fn reduction_add_is_reassociable(ctx: &FnCtx<'_>, left: &Expr, right: &Expr) -> } fn lower_arithmetic_operand(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<(String, bool)> { - // #10199: the element-shape fast clone's two NON-numeric reads — + // #10185: the element-shape fast clone's two NON-numeric reads — // `arr[i].name.length` and `arr[i].active ? 1 : 0`. Both are numbers by the // time they get here, each proven from the loaded word's own NaN-box tag // with a side exit to the slow clone on a miss, so the `true` suppresses diff --git a/crates/perry-codegen/src/expr/element_shape_guard.rs b/crates/perry-codegen/src/expr/element_shape_guard.rs index 7b199c3951..a8565b21b3 100644 --- a/crates/perry-codegen/src/expr/element_shape_guard.rs +++ b/crates/perry-codegen/src/expr/element_shape_guard.rs @@ -563,7 +563,7 @@ pub(crate) fn emit_element_shape_index( let slot = slot.clone(); Some(ctx.block().load(I32, &slot)) } - // #10199: the carried recurrence's private i32 slot, written by the + // #10185: the carried recurrence's private i32 slot, written by the // body's first statement earlier in this same iteration. The REAL // binding slot is deliberately NOT read here — it is one iteration // behind until the trailing write-back commits, which is exactly what @@ -642,14 +642,14 @@ pub(crate) fn emit_element_deref_with_residual( let ok = blk.and(I1, &hdr_ok, &shape_ok); // The side exit resumes the CURRENT iteration in the slow clone; no effect - // of this iteration has committed yet (#10199: with a prefetch this is the + // of this iteration has committed yet (#10185: with a prefetch this is the // ONLY residual exit in the iteration, and it precedes every store). blk.cond_br(&ok, &load_label, &fact.side_exit_label); ctx.current_block = load_idx; elem_handle } -/// #10199: the once-per-iteration element prologue. +/// #10185: the once-per-iteration element prologue. /// /// Emitted by the body's leading virtual binding (`stmt/let_stmt.rs` → /// `stmt/element_shape_loop::lower_virtual_clone_binding`) once the index for diff --git a/crates/perry-codegen/src/expr/element_shape_reads.rs b/crates/perry-codegen/src/expr/element_shape_reads.rs index ce207ae373..9d78d32174 100644 --- a/crates/perry-codegen/src/expr/element_shape_reads.rs +++ b/crates/perry-codegen/src/expr/element_shape_reads.rs @@ -1,4 +1,4 @@ -//! #10199: the two NON-numeric reads the element-shape fast clone admits — +//! #10185: the two NON-numeric reads the element-shape fast clone admits — //! `arr[i].prop.length` on a string field and `arr[i].prop ? A : B` on a //! boolean one. //! diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index ff54c5e34b..910ab2a688 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -2136,7 +2136,7 @@ pub(crate) enum ElementShapeIndex { /// Entry-block i32 alloca the clone writes the derived index to. slot: String, }, - /// #10199: a LOOP-CARRIED index — `let c = …;` outside the loop, + /// #10185: a LOOP-CARRIED index — `let c = …;` outside the loop, /// `c = (a*c + b) % m;` as the body's first statement, then `arr[c]` (or /// `const index = c; arr[index]`). The benchmark's `random` mode, and the /// shape every pseudo-random walk over a record array is written in. @@ -2151,7 +2151,7 @@ pub(crate) enum ElementShapeIndex { Carried(Box), } -/// #10199: everything [`ElementShapeIndex::Carried`] needs, boxed to keep the +/// #10185: everything [`ElementShapeIndex::Carried`] needs, boxed to keep the /// enum small. #[derive(Debug, Clone)] pub(crate) struct CarriedIndex { @@ -2174,7 +2174,7 @@ pub(crate) struct CarriedIndex { pub commit_slot: String, } -/// #10199: the per-iteration element prefetch. +/// #10185: the per-iteration element prefetch. /// /// `rows[index].id + rows[index].name.length + (rows[index].active ? 1 : 0)` /// reads ONE element three times. Without this each read repeats the element @@ -2220,7 +2220,7 @@ impl ElementShapeIndex { ) } - /// #10199: the locals whose `Let` / `LocalSet` the fast clone lowers + /// #10185: the locals whose `Let` / `LocalSet` the fast clone lowers /// VIRTUALLY for this index form. Nothing may read one of them bare inside /// the clone, and their shadow slots must not be cleared there. pub(crate) fn virtual_locals(&self) -> impl Iterator + '_ { @@ -2277,7 +2277,7 @@ pub(crate) struct ElementShapeLoopFact { pub index_local_id: u32, /// #10123: how the clone computes the element index. pub index: ElementShapeIndex, - /// #10199: the shared per-iteration element deref, when the body has a + /// #10185: the shared per-iteration element deref, when the body has a /// leading virtual binding to hang it off. `None` keeps #10123's per-read /// deref + residual check. pub elem_prefetch: Option, @@ -2310,7 +2310,7 @@ pub(crate) struct ElementShapeLoopFact { /// raw-f64 candidates; shape-keyed ones are the preheader's live query /// results (#10123). pub fields: std::collections::BTreeMap, - /// #10199: true when the fast clone lowers a body the matcher SYNTHESIZED + /// #10185: true when the fast clone lowers a body the matcher SYNTHESIZED /// (the K-statement accumulator fold, or the carried index's trailing /// write-back) rather than the source body. The function-wide /// shadow-slot-clear map is keyed by statement INDEX, so a rewritten body @@ -2394,7 +2394,7 @@ pub(crate) fn element_shape_loop_fact_for_property_get<'f>( (ElementShapeIndex::DerivedMod { local_id, .. }, Expr::LocalGet(id)) => { *id == *local_id } - // #10199: `arr[c]` and `const index = c; arr[index]` are + // #10185: `arr[c]` and `const index = c; arr[index]` are // the same subscript — the alias is virtual, so both // spellings read the carried slot the preheader bounded. (ElementShapeIndex::Carried(carried), Expr::LocalGet(id)) => { diff --git a/crates/perry-codegen/src/expr/shadow_slot.rs b/crates/perry-codegen/src/expr/shadow_slot.rs index 02c60d1bbe..2383e3890d 100644 --- a/crates/perry-codegen/src/expr/shadow_slot.rs +++ b/crates/perry-codegen/src/expr/shadow_slot.rs @@ -177,7 +177,7 @@ pub(crate) fn emit_shadow_slot_clear(ctx: &mut FnCtx<'_>, slot_idx: u32) { // reason: its `Let` emits one `srem` into a private i32 alloca, never a // shadow bind, so a lexical-death clear would be the clone's only call. // - // #10199's carried index (`c = (a*c + b) % m`) and its optional + // #10185's carried index (`c = (a*c + b) % m`) and its optional // `const index = c` alias are two more of the same. if ctx.element_shape_loop_facts.iter().any(|fact| { fact.element_binding @@ -187,7 +187,7 @@ pub(crate) fn emit_shadow_slot_clear(ctx: &mut FnCtx<'_>, slot_idx: u32) { }) { return; } - // #10199: a SYNTHESIZED fast-clone body (the accumulator fold, the carried + // #10185: a SYNTHESIZED fast-clone body (the accumulator fold, the carried // write-back) no longer has the statement indices the function-wide clear // map is keyed by, so every clear reached from inside it would be // attributed to some other statement's local. Suppress them for the fast diff --git a/crates/perry-codegen/src/stmt/element_shape_carried.rs b/crates/perry-codegen/src/stmt/element_shape_carried.rs index d6e8700436..3924d8e733 100644 --- a/crates/perry-codegen/src/stmt/element_shape_carried.rs +++ b/crates/perry-codegen/src/stmt/element_shape_carried.rs @@ -1,4 +1,4 @@ -//! #10199: the element-shape clone's LOOP-CARRIED index — the `random` access +//! #10185: the element-shape clone's LOOP-CARRIED index — the `random` access //! shape. //! //! ```text diff --git a/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs b/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs index c86fbd7e88..429523b2b5 100644 --- a/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs @@ -1,4 +1,4 @@ -//! #10199: the `fields` and `random` access shapes — IR census for the +//! #10185: the `fields` and `random` access shapes — IR census for the //! element-shape clone's loop-carried index, its multi-statement accumulator //! fold, and its two non-numeric reads. //! @@ -248,7 +248,7 @@ fn store_count(fast: &str) -> usize { } // --------------------------------------------------------------------------- -// #10199 — the LOOP-CARRIED index (`random`). +// #10185 — the LOOP-CARRIED index (`random`). // --------------------------------------------------------------------------- #[test] @@ -459,7 +459,7 @@ fn a_recurrence_over_a_second_variable_declines() { } // --------------------------------------------------------------------------- -// #10199 — the MULTI-STATEMENT body, the string `.length` and the ternary +// #10185 — the MULTI-STATEMENT body, the string `.length` and the ternary // (`fields`). // --------------------------------------------------------------------------- diff --git a/crates/perry-codegen/src/stmt/element_shape_loop.rs b/crates/perry-codegen/src/stmt/element_shape_loop.rs index 9bc43a8d17..47e54c7c46 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop.rs @@ -214,7 +214,7 @@ enum MatchedIndex { local_id: u32, modulus_id: u32, }, - /// #10199: `c = (a*c + b) % m; … arr[c]`, optionally spelled through a + /// #10185: `c = (a*c + b) % m; … arr[c]`, optionally spelled through a /// `const index = c` alias. The recurrence itself lives on /// [`ElementShapeVersionedLoop::carried`] — this arm records only the /// spellings the subscript may take. See `stmt/element_shape_carried.rs`. @@ -249,9 +249,9 @@ struct ElementShapeVersionedLoop { /// form; `None` for the original single-statement accumulator body. element_binding: Option, accumulator_id: u32, - /// #10199: the matched recurrence, for the [`MatchedIndex::Carried`] form. + /// #10185: the matched recurrence, for the [`MatchedIndex::Carried`] form. carried: Option, - /// #10199: the body the FAST clone lowers, when it is not the source body. + /// #10185: the body the FAST clone lowers, when it is not the source body. /// /// Two rewrites, both of which exist to put every side exit an iteration /// can take BEFORE any of its stores: @@ -275,7 +275,7 @@ struct PureExprScope { /// #10123: `(d, m)` for a body whose first statement is /// `const d = counter % m`. derived: Option<(u32, u32)>, - /// #10199: `(carried, alias)` for a body whose first statement advances a + /// #10185: `(carried, alias)` for a body whose first statement advances a /// loop-carried index. Both are excluded from bare reads for the same /// reason `derived` is — inside the clone the real slot is one iteration /// behind until the trailing commit. @@ -345,7 +345,7 @@ fn element_shape_loop_pure_expr_collect( out.props.insert(property.clone()); true } - // #10199: `arr[].prop.length` — a STRING field's JS length. + // #10185: `arr[].prop.length` — a STRING field's JS length. // Admitted as its own form rather than as "a read of `prop` // followed by a generic `.length`", because the generic one is a // property diamond ending in a runtime call, and a call inside this @@ -357,7 +357,7 @@ fn element_shape_loop_pure_expr_collect( } _ => false, }, - // #10199: `arr[].prop ? A : B` with constant arms. JS truthiness + // #10185: `arr[].prop ? A : B` with constant arms. JS truthiness // of an arbitrary value is a runtime question the clone is not allowed // to guess, so the emitted read admits ONLY the two boolean singletons // and side-exits on everything else. Anything but a tracked element @@ -383,7 +383,7 @@ fn element_shape_loop_pure_expr_collect( Expr::LocalGet(id) => { scope.element_binding != Some(*id) && scope.derived.map(|(d, _)| d) != Some(*id) - // #10199: the carried index and its alias are virtual too — + // #10185: the carried index and its alias are virtual too — // their real slots are one iteration behind inside the clone, // so a bare read would hand out a stale value. && scope.carried.is_none_or(|(c, alias)| *id != c && alias != Some(*id)) @@ -444,7 +444,7 @@ fn match_index_form(index: &perry_hir::Expr, scope: &PureExprScope) -> Option match scope.carried { @@ -695,7 +695,7 @@ fn anon_shape_field_type_is_compatible( /// clone, never correctness. pub(super) fn lower_virtual_clone_binding(ctx: &mut FnCtx<'_>, id: u32) -> Result { enum VirtualBinding { - /// #7771 `const r = arr[j]` and #10199 `const index = c`: pure aliases, + /// #7771 `const r = arr[j]` and #10185 `const index = c`: pure aliases, /// no instruction of their own. Alias, /// #10123 `const d = j % m`: one `srem i32`. @@ -748,13 +748,13 @@ pub(super) fn lower_virtual_clone_binding(ctx: &mut FnCtx<'_>, id: u32) -> Resul blk.store(I32, &derived, &slot); } } - // #10199: this iteration's index is now in its slot, so this is where the + // #10185: this iteration's index is now in its slot, so this is where the // shared element deref belongs — for the ONE binding that owns it. emit_element_prefetch_for(ctx, scope_id, id); Ok(true) } -/// #10199: emit the once-per-iteration element prologue, if `site_local` is the +/// #10185: emit the once-per-iteration element prologue, if `site_local` is the /// statement the fact designated to own it. /// /// Every read in the clone loads the handle this parks, so the designated @@ -941,7 +941,7 @@ fn match_element_shape_versioned_loop( // 3. `const d = j % m; acc = ` — // #10123's derived-index form, the shape every sequential pass over a // parsed record array is written in; - // 4. #10199's loop-carried form, `c = (a*c + b) % m;` optionally followed + // 4. #10185's loop-carried form, `c = (a*c + b) % m;` optionally followed // by `const index = c;`. // // In 2, 3 and 4 the leading statement is VIRTUAL inside the fast clone: it @@ -954,7 +954,7 @@ fn match_element_shape_versioned_loop( // value. Form 4's `c` is the one mutable exception, and it pays for it with // a write-back (see `element_shape_carried`). // - // #10199 also admits K >= 1 accumulator statements instead of exactly one, + // #10185 also admits K >= 1 accumulator statements instead of exactly one, // which is what `sum += rows[i].id; sum += rows[i].name.length; …` needs. // NOTHING else is admitted. let mut element_binding: Option<(u32, u32)> = None; @@ -1034,7 +1034,7 @@ fn match_element_shape_versioned_loop( } derived = Some((*id, *modulus_id)); } - // #10199: `const index = c` after the recurrence. A pure alias, so + // #10185: `const index = c` after the recurrence. A pure alias, so // it emits nothing at all — the subscript resolves to the carried // slot either way. Expr::LocalGet(aliased) => { @@ -1072,7 +1072,7 @@ fn match_element_shape_versioned_loop( { return None; } - // #10199: statements 2..K are folded into statement 1 for the fast clone + // #10185: statements 2..K are folded into statement 1 for the fast clone // (`acc = a; acc = acc + b` ≡ `acc = (a) + b`), so that the WHOLE iteration // commits once, after every side exit it can take. Each of them must // therefore be `acc ` with `acc` as the left operand — the shape @@ -1134,7 +1134,7 @@ fn match_element_shape_versioned_loop( return None; } } - // #10199: a matched recurrence whose value nothing subscripts with would + // #10185: a matched recurrence whose value nothing subscripts with would // still have its `LocalSet` lowered generically — a `frem` libcall inside // the clone, which deletes it. Decline instead, so the loop keeps whatever // lowering it has today. @@ -1223,7 +1223,7 @@ fn match_element_shape_versioned_loop( None => return None, }; - // The property denylist, which is arm-specific (#10199). + // The property denylist, which is arm-specific (#10185). // // The CLASS arm's list exists because its read bakes in a compile-time // packed slot index while the surrounding lowering may route the name @@ -1278,7 +1278,7 @@ fn match_element_shape_versioned_loop( return None; } - // #10199: the fast clone's body, when the two rewrites apply. `lead` is + // #10185: the fast clone's body, when the two rewrites apply. `lead` is // whatever statements the walk consumed ahead of the accumulator run (the // recurrence and/or the virtual binding), kept verbatim. let lead = &body[..body.len() - rest.len()]; @@ -1497,7 +1497,7 @@ pub(super) fn lower_element_shape_versioned_for( // in JS, so a zero modulus is a slow-clone case rather than something the // clone may compute. // - // #10199's carried recurrence takes the same obligation for the same + // #10185's carried recurrence takes the same obligation for the same // reason, and one more of its own: its ENTRY value must be a non-negative // integral i32, because the whole i64 recurrence bound // (`a * i32::MAX + b < 2^53`, and a non-negative dividend for `srem` to @@ -1555,7 +1555,7 @@ pub(super) fn lower_element_shape_versioned_for( (MatchedIndex::Constant(k), _) => { crate::expr::element_shape_guard::ElementShapeIndexBound::Constant(*k) } - // #10199: the recurrence's result is `srem` of a non-negative dividend + // #10185: the recurrence's result is `srem` of a non-negative dividend // by `m`, so it lands in `[0, m)` exactly like the derived index and // takes exactly the same preheader obligation, `m <= length`. (MatchedIndex::DerivedMod { .. } | MatchedIndex::Carried { .. }, Some(modulus)) => { @@ -1684,10 +1684,10 @@ pub(super) fn lower_element_shape_versioned_for( } }; - // #10199: the shared once-per-iteration element deref, hung off the body's + // #10185: the shared once-per-iteration element deref, hung off the body's // leading virtual statement. Shape-keyed only: the class-keyed arm's reads // are raw doubles behind a `GC_OBJ_TYPED_LAYOUT_INTACT` residual whose - // emitted IR #7480's census pins, and nothing in #10199 measures it. + // emitted IR #7480's census pins, and nothing in #10185 measures it. let elem_prefetch = shape_keyed .then(|| { let site_local_id = match (&fact_index, matched.element_binding) { diff --git a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs index fcbaa46dfb..20d71b035a 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs @@ -402,7 +402,7 @@ fn assert_fast_clone_is_entered(ir: &str) { fn fast_clone_slice(ir: &str) -> String { let mut owned = String::new(); let mut in_fast_block = false; - // #10199: the emitted text carries the function TWICE (same block labels), + // #10185: the emitted text carries the function TWICE (same block labels), // so every block would be collected twice and any `matches().count()` // assertion against the slice would read double. Stop at the first repeated // label, which is where the second copy begins — `contains` assertions are @@ -414,7 +414,7 @@ fn fast_clone_slice(ir: &str) -> String { // A block DEFINITION starts at column 0 and ends in `:`; anything else // belongs to whichever block was last opened. if !line.starts_with(char::is_whitespace) && trimmed.ends_with(':') { - // #10199 added three more: the string-`.length` decode + // #10185 added three more: the string-`.length` decode // (`element_shape.strlen*`), the boolean ternary's admitted arm // (`element_shape.bool`), and nothing for the shared prefetch, // which reuses `element_shape.load`. Every block the clone can diff --git a/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs b/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs index fe47ad0e89..6d5d265332 100644 --- a/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs @@ -451,7 +451,7 @@ fn an_untracked_local_index_declines() { #[test] fn the_shape_keyed_arm_denies_only_proto() { - // #10199 narrowed this list. The CLASS arm still denies every name with a + // #10185 narrowed this list. The CLASS arm still denies every name with a // dedicated branch in the property dispatch, because its read bakes in a // compile-time packed slot. The SHAPE arm bakes in nothing: the preheader // asks the runtime for that exact key's inline slot in that exact ordinary @@ -547,7 +547,7 @@ fn a_declared_but_unresolvable_element_type_still_declines() { ); } -/// #10199's `fields` and `random` shapes — a child module so it inherits both +/// #10185's `fields` and `random` shapes — a child module so it inherits both /// this file's shape-keyed assertions and `element_shape_loop_tests`'s slicing /// helpers. #[path = "element_shape_fields_random_tests.rs"] diff --git a/crates/perry-codegen/src/stmt/mod.rs b/crates/perry-codegen/src/stmt/mod.rs index 5b9ac4f33e..cc27d26763 100644 --- a/crates/perry-codegen/src/stmt/mod.rs +++ b/crates/perry-codegen/src/stmt/mod.rs @@ -282,7 +282,7 @@ fn lower_return_expr(ctx: &mut FnCtx<'_>, expr: &perry_hir::Expr) -> Result, stmt: &Stmt) -> Result<()> { match stmt { Stmt::Expr(e) => { - // #10199: the element-shape fast clone's carried-index statements + // #10185: the element-shape fast clone's carried-index statements // (the recurrence and its trailing write-back) are lowered // VIRTUALLY, exactly like the `Let` bindings in `let_stmt.rs` — // the generic lowering of `c = (c * 17 + 7) % length` is an diff --git a/crates/perry-codegen/src/type_analysis/numeric.rs b/crates/perry-codegen/src/type_analysis/numeric.rs index e3c20605f2..338442e398 100644 --- a/crates/perry-codegen/src/type_analysis/numeric.rs +++ b/crates/perry-codegen/src/type_analysis/numeric.rs @@ -356,7 +356,7 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { Expr::Logical { left, right, .. } => { is_numeric_expr(ctx, left) && is_numeric_expr(ctx, right) } - // #10199: `arr[i]. ? 1 : 0` inside an element-shape fast + // #10185: `arr[i]. ? 1 : 0` inside an element-shape fast // clone. Deliberately NOT the general "both arms are numeric" rule — a // general ternary's CONDITION is an arbitrary JS truthiness test, which // is a runtime call; this arm is true only for the exact shape the @@ -386,7 +386,7 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { if property == "length" && expression_has_numeric_length(ctx, object) { return true; } - // #10199: `arr[i]..length` inside an element-shape + // #10185: `arr[i]..length` inside an element-shape // fast clone. The receiver is an untyped element read, so the // declared-type answer above cannot see a string there; the clone's // own lowering tag-tests the loaded word for both string diff --git a/test-files/test_gap_json_record_loop_clone.ts b/test-files/test_gap_json_record_loop_clone.ts index 0a60a97359..363fb7a04b 100644 --- a/test-files/test_gap_json_record_loop_clone.ts +++ b/test-files/test_gap_json_record_loop_clone.ts @@ -270,3 +270,301 @@ function sumTwoFields(rows: any, count: number): number { return sum; } console.log("two-fields:", sumTwoFields(moduleRows, moduleLength)); + +// --------------------------------------------------------------------------- +// 9. #10185: the LOOP-CARRIED index (`random`), the K-statement accumulator +// fold, and the two non-numeric reads. +// +// Every case below is a MISCOMPILE if it is wrong, not a slow path: +// +// * the carried write-back is placed at the END of the iteration, so a +// mid-iteration side exit leaves the real binding holding that +// iteration's ENTRY value and the slow clone advances the recurrence +// exactly once. A write-back at the update site double-applies it, and +// every later index is silently a different (still valid) record; +// * K accumulator statements fold to one store for the same reason: a +// side exit from the third read must not leave the first two applied; +// * `.length` and the ternary read a NaN-boxed word and assume a +// representation, so each tag-tests it and side-exits otherwise. +// --------------------------------------------------------------------------- + +// The benchmark's `random` mode, with the `const index = cursor` alias. +function randomSum(rows: any, count: number, n: number): number { + let sum = 0; + let cursor = 0; + for (let i = 0; i < count; i++) { + cursor = (cursor * 17 + 7) % n; + const index = cursor; + sum += rows[index].id; + } + return sum; +} + +// The same recurrence subscripted DIRECTLY, with the carried value read after +// the loop — the clone owes a write-back, and this is what observes it. +function randomSumAndCursor( + rows: any, + count: number, + n: number, + start: number, +): string { + let sum: any = 0; + let cursor = start; + for (let i = 0; i < count; i++) { + cursor = (cursor * 17 + 7) % n; + sum += rows[cursor].id; + } + return String(sum) + "|" + String(cursor); +} + +// A multiplier large enough that `a * cursor` leaves the range an `f64` +// represents exactly. JavaScript evaluates this in doubles, so an i64 +// recurrence would NOT agree — the matcher must decline and let the ordinary +// lowering run. +function randomSumWideMultiplier( + rows: any, + count: number, + n: number, +): string { + let sum = 0; + let cursor = 3; + for (let i = 0; i < count; i++) { + cursor = (cursor * 600000000 + 7) % n; + sum += rows[cursor].id; + } + return String(sum) + "|" + String(cursor); +} + +// A recurrence with an even step, so a fractional carried value still lands +// on whole-number indices after the first update. +function randomSumEvenStep( + rows: any, + count: number, + n: number, + start: number, +): string { + let sum = 0; + let cursor = start; + for (let i = 0; i < count; i++) { + cursor = (cursor * 2 + 7) % n; + sum += rows[cursor].id; + } + return String(sum) + "|" + String(cursor); +} + +// The benchmark's `fields` mode: three accumulator statements over ONE +// element, one through a string and one through a boolean. +function fieldsSum(rows: any, count: number, n: number): number { + let sum = 0; + for (let i = 0; i < count; i++) { + const index = i % n; + sum += rows[index].id; + sum += rows[index].name.length; + sum += rows[index].active ? 1 : 0; + } + return sum; +} + +const randomRows: any = JSON.parse(buildRecords(40, "pad")); +console.log("random-sum:", randomSum(randomRows, 200, randomRows.length)); +console.log( + "random-sum-again:", + randomSum(randomRows, 200, randomRows.length), +); +console.log( + "random-cursor-live-after:", + randomSumAndCursor(randomRows, 200, randomRows.length, 0), +); +// An odd trip count observes a different point of the cycle than the run +// above, so the write-back is checked at two places rather than one. +console.log( + "random-cursor-live-after-odd:", + randomSumAndCursor(randomRows, 201, randomRows.length, 0), +); +// A modulus smaller than the array. 13 is chosen because `(17c + 7) % 13` +// walks six distinct indices from 0 — several plausible moduli make the +// recurrence a fixed point at 0, where every assertion below would hold of a +// loop that read one element forever. +console.log( + "random-small-modulus:", + randomSumAndCursor(randomRows, 50, 13, 0), +); +console.log( + "random-modulus-one:", + randomSumAndCursor(randomRows, 9, 1, 0), +); + +// Entry values the preheader must reject: JS `%` returns a NEGATIVE remainder +// for a negative dividend, and a fractional index is not an index at all. +// Both take the slow clone and keep ordinary JavaScript semantics. +try { + console.log( + "random-negative-start:", + randomSumAndCursor(randomRows, 4, 5, -3), + ); +} catch (err) { + console.log("random-negative-start-threw:", String(err).slice(0, 9)); +} +// A FRACTIONAL entry value. The step is even so every index the loop +// actually reads is a whole number — the divergence being checked is the +// preheader's, not the subscript's: if the clone materialized `0.5` as the +// i32 `0` it would start the recurrence one step early and every index would +// shift, which the sum and the final cursor both show. +console.log( + "random-fractional-start:", + randomSumEvenStep(randomRows, 6, 5, 0.5), +); +console.log( + "random-integral-start-same-step:", + randomSumEvenStep(randomRows, 6, 5, 0), +); +// `x % 0` is NaN, and `rows[NaN]` is `undefined`. +try { + console.log("random-modulus-zero:", randomSumAndCursor(randomRows, 3, 0, 0)); +} catch (err) { + console.log("random-modulus-zero-threw:", String(err).slice(0, 9)); +} +// A modulus past the array's length runs off the end. +try { + console.log( + "random-modulus-past-length:", + randomSumAndCursor(randomRows, 30, 500, 0), + ); +} catch (err) { + console.log("random-modulus-past-length-threw:", String(err).slice(0, 9)); +} +console.log( + "random-wide-multiplier:", + randomSumWideMultiplier(randomRows, 25, 11), +); + +// THE write-back test. The array is homogeneous in SHAPE, so the clone is +// entered — but one `id` is a string, so the per-read Number tag test side +// -exits mid-iteration and the slow clone re-runs it. If the recurrence had +// already been committed, the slow clone advances it a SECOND time and both +// the sum and the final cursor drift. +// `(17c + 7) % 5` walks 0 -> 2 -> 1 -> 4 -> 0, so index 2 — the string one — +// really is read. A modulus that skipped it would make this whole case +// vacuous while still printing a matching number. +const randomMixed: any = JSON.parse( + '[{"id":0},{"id":1},{"id":"two"},{"id":3},{"id":4}]', +); +console.log( + "random-side-exit:", + randomSumAndCursor(randomMixed, 30, randomMixed.length, 0), +); +console.log( + "random-side-exit-again:", + randomSumAndCursor(randomMixed, 30, randomMixed.length, 0), +); + +// The recurrence over a HETEROGENEOUS array declines at the array level. +console.log( + "random-hetero:", + randomSumAndCursor(hetero, 16, hetero.length, 0), +); + +// --------------------------------------------------------------------------- +// 9b. The `fields` shape. +// --------------------------------------------------------------------------- + +// SSO names (<= 5 bytes, packed into the NaN-box) and heap names in the SAME +// array — the shape is identical either way, and the `.length` decode has to +// handle both. +const mixedNames: any = JSON.parse( + '[{"id":1,"name":"ab","active":true},' + + '{"id":2,"name":"user_222","active":false},' + + '{"id":3,"name":"","active":true},' + + '{"id":4,"name":"abcde","active":false},' + + '{"id":5,"name":"abcdef","active":true}]', +); +console.log("fields-sso-and-heap:", fieldsSum(mixedNames, 25, mixedNames.length)); +console.log( + "fields-sso-and-heap-again:", + fieldsSum(mixedNames, 25, mixedNames.length), +); + +// A NON-STRING `name`: `.length` of a number is `undefined`, so the sum goes +// NaN — and the clone must side-exit rather than decode a double as a string +// header. The fold is what keeps the earlier `id` from being applied twice. +const numberName: any = JSON.parse( + '[{"id":1,"name":"ab","active":true},{"id":2,"name":7,"active":true},' + + '{"id":3,"name":"cd","active":false}]', +); +console.log("fields-number-name:", fieldsSum(numberName, 9, numberName.length)); + +// A NULL `name`: `null.length` throws, exactly as JS says. +const nullName: any = JSON.parse( + '[{"id":1,"name":"ab","active":true},{"id":2,"name":null,"active":true}]', +); +try { + console.log("fields-null-name:", fieldsSum(nullName, 4, nullName.length)); +} catch (err) { + console.log("fields-null-name-threw:", String(err).slice(0, 9)); +} + +// NON-BOOLEAN `active` values. JS truthiness of `0`, `""`, `"no"` and `null` +// is a runtime question the clone is not allowed to guess: only the two +// boolean singletons are admitted and everything else side-exits. +const truthyActive: any = JSON.parse( + '[{"id":1,"name":"ab","active":true},{"id":2,"name":"cd","active":0},' + + '{"id":3,"name":"ef","active":"no"},{"id":4,"name":"gh","active":""},' + + '{"id":5,"name":"ij","active":null},{"id":6,"name":"kl","active":false}]', +); +console.log( + "fields-truthiness:", + fieldsSum(truthyActive, 24, truthyActive.length), +); +console.log( + "fields-truthiness-again:", + fieldsSum(truthyActive, 24, truthyActive.length), +); + +// Two accumulator statements where the SECOND read side-exits: the first must +// not be applied twice when the slow clone re-runs the iteration. +function twoStatementSum(rows: any, count: number, n: number): number { + let sum = 0; + for (let i = 0; i < count; i++) { + const index = i % n; + sum += rows[index].id; + sum += rows[index].score; + } + return sum; +} +const secondFieldMixed: any = JSON.parse( + '[{"id":1,"score":10},{"id":2,"score":"x"},{"id":3,"score":30}]', +); +console.log( + "fold-side-exit:", + twoStatementSum(secondFieldMixed, 9, secondFieldMixed.length), +); + +// Heterogeneous shapes with the multi-read body. +console.log("fields-hetero:", fieldsSum(hetero, 12, hetero.length)); + +// --------------------------------------------------------------------------- +// 9c. Property names the CLASS-keyed arm denies. A parsed record's own data +// property SHADOWS the prototype name it collides with, and the +// shape-keyed preheader resolves it from the runtime shape table, so +// `name` and `length` are ordinary inline slots here. `name` in +// particular is what the access benchmark's own `fields` mode reads. +// --------------------------------------------------------------------------- +function ownNamedFields(rows: any, count: number, n: number): number { + let sum = 0; + for (let i = 0; i < count; i++) { + const index = i % n; + sum += rows[index].length; + sum += rows[index].size; + sum += rows[index].name.length; + } + return sum; +} +const ownNames: any = JSON.parse( + '[{"length":3,"size":4,"name":"ab"},{"length":5,"size":6,"name":"cdef"},' + + '{"length":7,"size":8,"name":"ghijklm"}]', +); +console.log("own-named-fields:", ownNamedFields(ownNames, 12, ownNames.length)); +console.log( + "own-named-fields-again:", + ownNamedFields(ownNames, 12, ownNames.length), +); From c7c3da945162dc94615e795cad74aa876af089d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 13:03:18 +0200 Subject: [PATCH 07/10] lint(census): follow the element clone's ShapeId read to its new emitter The shape-descriptor census asserts that the element-shape guard reads the authoritative ShapeId at header offset 4. That read moved out of `emit_element_shape_field_load` into the deref helper the per-read path and the shared prologue now share, so the census names the function that actually emits it. (cherry picked from commit 54765a927ba8a6a8c7cf0a7c52aca885b1c8d514) --- scripts/shape_descriptor_census.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/shape_descriptor_census.py b/scripts/shape_descriptor_census.py index 9815d926e0..18b85fd2e4 100644 --- a/scripts/shape_descriptor_census.py +++ b/scripts/shape_descriptor_census.py @@ -652,7 +652,13 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: "emit_proven_shape_recheck", "emit_class_field_inline_precheck", )), - (raw_element_guard, ("emit_element_shape_field_load",)), + # #10185: the element clone's residual moved out of + # `emit_element_shape_field_load` into the deref helper both the + # per-read path and the shared once-per-iteration prologue call, so the + # ShapeId read this asserts now lives there. Naming the function that + # actually emits the check is what keeps the assertion from going + # vacuous the next time the emitter is split. + (raw_element_guard, ("emit_element_deref_with_residual",)), ): for name in names: body = function_body(source, name) From 31732e32c772054e2a4b47607756e9fbb401d4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 13:33:17 +0200 Subject: [PATCH 08/10] docs(changelog): record the measured fields/random numbers 12-cell interleaved A/B against node 26.5.1 and bun 1.3.14, plus instructions retired per iteration, all from one coherent build of this worktree at the base commit and at HEAD. The one cell that does not reach parity (16k fields, 1.029x) is named rather than rounded off. (cherry picked from commit 533b59247bc7b5da51201b365d9dfa93f1698733) --- .../10185-element-shape-fields-random.md | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/changelog.d/10185-element-shape-fields-random.md b/changelog.d/10185-element-shape-fields-random.md index 788c133fdf..07b0533c87 100644 --- a/changelog.d/10185-element-shape-fields-random.md +++ b/changelog.d/10185-element-shape-fields-random.md @@ -94,24 +94,25 @@ built from this worktree, `base` at #10171's head `2b77e7d4fe`): | cell | base (#10171) | new | node 26.5.1 | bun 1.3.14 | new / best | |---|---:|---:|---:|---:|---:| -| 16k `repeat` | 4.15 | 4.19 | 3.24 | 5.03 | 1.293 | -| 16k `sequential` | 4.21 | 4.17 | 4.35 | 6.14 | 0.959 | -| 16k `random` | 15.38 | **4.99** | 7.10 | 9.61 | **0.703** | -| 16k `fields` | 25.76 | **6.19** | 5.75 | 8.35 | **1.077** | -| 1m `repeat` | 4.12 | 4.19 | 3.57 | 5.25 | 1.174 | -| 1m `sequential` | 4.24 | 4.18 | 9.35 | 8.06 | 0.519 | -| 1m `random` | 15.90 | **5.25** | 12.66 | 11.66 | **0.450** | -| 1m `fields` | 29.02 | **6.31** | 12.68 | 15.41 | **0.498** | -| 20m `repeat` | 4.19 | 4.17 | 3.52 | 5.13 | 1.185 | -| 20m `sequential` | 4.29 | 4.37 | 7.48 | 8.83 | 0.584 | -| 20m `random` | 27.56 | **5.60** | 9.45 | 10.64 | **0.593** | -| 20m `fields` | 27.26 | **6.22** | 11.53 | 14.93 | **0.539** | - -The two targeted shapes move 3.1x-4.9x and land below the better of node and +| 16k `repeat` | 4.21 | 4.17 | 3.56 | 5.23 | 1.171 | +| 16k `sequential` | 4.17 | 4.30 | 4.57 | 6.25 | 0.941 | +| 16k `random` | 15.48 | **4.97** | 7.16 | 9.96 | **0.694** | +| 16k `fields` | 25.59 | **6.05** | 5.88 | 9.77 | **1.029** | +| 1m `repeat` | 4.15 | 4.17 | 3.19 | 5.05 | 1.307 | +| 1m `sequential` | 4.26 | 4.20 | 9.42 | 7.96 | 0.528 | +| 1m `random` | 16.00 | **5.32** | 11.71 | 11.17 | **0.476** | +| 1m `fields` | 28.14 | **6.17** | 12.70 | 14.91 | **0.486** | +| 20m `repeat` | 4.12 | 4.11 | 3.50 | 5.24 | 1.174 | +| 20m `sequential` | 4.20 | 4.19 | 6.98 | 8.51 | 0.600 | +| 20m `random` | 27.53 | **5.63** | 9.31 | 10.56 | **0.605** | +| 20m `fields` | 27.05 | **6.06** | 10.98 | 14.81 | **0.552** | + +The two targeted shapes move 3.0x-4.9x and land below the better of node and bun on five of their six cells. The sixth, 16k `fields`, is the one that does -not reach parity: 6.08 against node's 5.83 on a 12-round re-measure (1.043x). +not reach parity: 6.05 against node's 5.88 (1.029x; 6.08 vs 5.83 on a +separate 12-round re-measure of just that pair). That is the smallest fixture, where the whole array is in L1 and the guard -overhead is the only thing left — node's own `fields` is just 1.34x its +overhead is the only thing left — node's own `fields` is just 1.29x its `sequential` there, because its inline caches have type feedback proving `name` is a string and `active` a boolean, while this clone tag-tests both on every read. Paying that test is what lets it side-exit instead of deoptimize, so it @@ -128,10 +129,10 @@ Instructions retired per iteration on the 16k fixture (2M iterations minus a | shape | base | new | |---|---:|---:| -| random | 207.9 | 29.1 | -| fields | 466.6 | 47.0 | -| sequential | 25.1 | 25.1 | -| repeat | 16.1 | 15.8 | +| random | 207.6 | 29.1 | +| fields | 466.4 | 47.0 | +| sequential | 25.0 | 25.6 | +| repeat | 16.0 | 16.1 | `random` lands one recurrence above `sequential`'s 25, and `fields` two extra guarded reads plus a string decode and a select above it — which is what the From c173226a0a5729405cae2d24b81fc86800935b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 14:47:42 +0200 Subject: [PATCH 09/10] test(runtime): mint the prefix-test meta record under a GC suppress scope (#10196) --- .../src/object/field_get_set/ic_miss/ic_slow.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 4f36ea44aa..00d62f64d8 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -510,8 +510,11 @@ mod tests { }); // Give the receiver a meta record and an arbitrary prefix token. const TOKEN: u64 = 0xA11CE; - let (meta, _) = obj.across_mut::(|| unsafe { - crate::object::object_meta_ensure(obj.get_raw_mut_ptr::()) + // Collections are suppressed around the allocating meta mint, so the + // receiver cannot move while the scoped pointer is live. + let meta = obj.with_mut_ptr(|o: *mut ObjectHeader| { + let _no_gc = crate::gc::GcSuppressScope::new(); + unsafe { crate::object::object_meta_ensure(o) } }); assert!( !meta.is_null(), From 058a509a280de8cc69bba2adf383f080e97e543c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 17:18:32 +0200 Subject: [PATCH 10/10] chore: bump workspace version to 0.5.1557 --- CLAUDE.md | 2 +- Cargo.lock | 162 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 3 files changed, 83 insertions(+), 83 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ff9b0bdad8..2a074af4fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1556 +**Current Version:** 0.5.1557 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 1de24a4505..e91294a5ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5690,7 +5690,7 @@ checksum = "1542e48011813fbdf3c075da4a4ed53ee93c816eef62e36eb5064a6fd2be10a5" [[package]] name = "perry" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "base64 0.22.1", @@ -5754,7 +5754,7 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-dispatch", "serde", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-audio-miniaudio" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "cc", "libc", @@ -5771,7 +5771,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "aho-corasick", "anyhow", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-dispatch", @@ -5814,7 +5814,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -5822,7 +5822,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "base64 0.22.1", @@ -5834,7 +5834,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -5842,7 +5842,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "async-trait", @@ -5870,14 +5870,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "serde", "serde_json", @@ -5885,7 +5885,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1556" +version = "0.5.1557" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "clap", @@ -5911,7 +5911,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "block2", "objc2", @@ -5921,7 +5921,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "argon2", "perry-ffi", @@ -5930,7 +5930,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "reqwest", @@ -5939,7 +5939,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bcrypt", "perry-ffi", @@ -5947,7 +5947,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "rusqlite", @@ -5955,7 +5955,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "scraper", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-runtime", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "chrono", "cron", @@ -5981,7 +5981,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "chrono", "perry-ffi", @@ -5989,7 +5989,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "rust_decimal", @@ -5997,7 +5997,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "serde_json", @@ -6005,7 +6005,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "rand 0.10.2", @@ -6013,7 +6013,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-runtime", @@ -6021,14 +6021,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bytes", "http-body-util", @@ -6046,7 +6046,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bytes", "lazy_static", @@ -6059,7 +6059,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "bytes", @@ -6091,7 +6091,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "lazy_static", "perry-ffi", @@ -6101,7 +6101,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "jsonwebtoken", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "lru", "perry-ffi", @@ -6121,7 +6121,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "chrono", "perry-ffi", @@ -6129,7 +6129,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bson", "futures-util", @@ -6141,7 +6141,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "chrono", "perry-ffi", @@ -6153,7 +6153,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "nanoid", "perry-ffi", @@ -6162,7 +6162,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bytes", "perry-ffi", @@ -6177,7 +6177,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "const-oid 0.10.2", "der 0.8.1", @@ -6196,7 +6196,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "lettre", "perry-ffi", @@ -6206,7 +6206,7 @@ dependencies = [ [[package]] name = "perry-ext-parcel-watcher" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "notify", "perry-ffi", @@ -6218,7 +6218,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "printpdf", @@ -6226,7 +6226,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "sqlx", @@ -6235,7 +6235,7 @@ dependencies = [ [[package]] name = "perry-ext-qs" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-runtime", @@ -6244,7 +6244,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "governor", "perry-ffi", @@ -6252,7 +6252,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "fast_image_resize", "image", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "perry-ext-streams" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "lazy_static", "perry-ffi", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "perry-ext-typescript" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-ffi", @@ -6292,7 +6292,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-runtime", @@ -6301,7 +6301,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "uuid", @@ -6309,7 +6309,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-validation", @@ -6318,7 +6318,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "futures-util", "lazy_static", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "brotli", "flate2", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "dashmap 6.2.1", "once_cell", @@ -6351,7 +6351,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-api-manifest", @@ -6372,11 +6372,11 @@ dependencies = [ [[package]] name = "perry-native-registration" -version = "0.5.1556" +version = "0.5.1557" [[package]] name = "perry-parser" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-diagnostics", @@ -6390,7 +6390,7 @@ dependencies = [ [[package]] name = "perry-perex" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perex", "regex", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "ahash", "anyhow", @@ -6458,14 +6458,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6560,14 +6560,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -6576,7 +6576,7 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-ui-model", @@ -6584,7 +6584,7 @@ dependencies = [ [[package]] name = "perry-ui-android" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "itoa", @@ -6602,7 +6602,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "rand 0.10.2", "serde", @@ -6612,7 +6612,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "cairo-rs 0.22.9", @@ -6635,7 +6635,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "block2", @@ -6652,7 +6652,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "block2", @@ -6669,7 +6669,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1556" +version = "0.5.1557" [[package]] name = "perry-ui-test" @@ -6680,11 +6680,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1556" +version = "0.5.1557" [[package]] name = "perry-ui-tvos" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "block2", @@ -6701,7 +6701,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "block2", @@ -6718,7 +6718,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "block2", "libc", @@ -6732,7 +6732,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "libc", @@ -6751,7 +6751,7 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "libc", @@ -6764,7 +6764,7 @@ dependencies = [ [[package]] name = "perry-updater" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "base64 0.22.1", @@ -6780,7 +6780,7 @@ dependencies = [ [[package]] name = "perry-validation" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "idna", "regex", @@ -6790,7 +6790,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index 5d704de8b7..9b3dfc8fa6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,7 +338,7 @@ codegen-units = 1 codegen-units = 1 [workspace.package] -version = "0.5.1556" +version = "0.5.1557" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry"