diff --git a/CLAUDE.md b/CLAUDE.md index ff9b0bdad8..2a074af4fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1556 +**Current Version:** 0.5.1557 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 1de24a4505..e91294a5ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5690,7 +5690,7 @@ checksum = "1542e48011813fbdf3c075da4a4ed53ee93c816eef62e36eb5064a6fd2be10a5" [[package]] name = "perry" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "base64 0.22.1", @@ -5754,7 +5754,7 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-dispatch", "serde", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-audio-miniaudio" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "cc", "libc", @@ -5771,7 +5771,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "aho-corasick", "anyhow", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-dispatch", @@ -5814,7 +5814,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -5822,7 +5822,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "base64 0.22.1", @@ -5834,7 +5834,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -5842,7 +5842,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "async-trait", @@ -5870,14 +5870,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "serde", "serde_json", @@ -5885,7 +5885,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1556" +version = "0.5.1557" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "clap", @@ -5911,7 +5911,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "block2", "objc2", @@ -5921,7 +5921,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "argon2", "perry-ffi", @@ -5930,7 +5930,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "reqwest", @@ -5939,7 +5939,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bcrypt", "perry-ffi", @@ -5947,7 +5947,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "rusqlite", @@ -5955,7 +5955,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "scraper", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-runtime", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "chrono", "cron", @@ -5981,7 +5981,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "chrono", "perry-ffi", @@ -5989,7 +5989,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "rust_decimal", @@ -5997,7 +5997,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "serde_json", @@ -6005,7 +6005,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "rand 0.10.2", @@ -6013,7 +6013,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-runtime", @@ -6021,14 +6021,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bytes", "http-body-util", @@ -6046,7 +6046,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bytes", "lazy_static", @@ -6059,7 +6059,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "bytes", @@ -6091,7 +6091,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "lazy_static", "perry-ffi", @@ -6101,7 +6101,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "jsonwebtoken", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "lru", "perry-ffi", @@ -6121,7 +6121,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "chrono", "perry-ffi", @@ -6129,7 +6129,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bson", "futures-util", @@ -6141,7 +6141,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "chrono", "perry-ffi", @@ -6153,7 +6153,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "nanoid", "perry-ffi", @@ -6162,7 +6162,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "bytes", "perry-ffi", @@ -6177,7 +6177,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "const-oid 0.10.2", "der 0.8.1", @@ -6196,7 +6196,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "lettre", "perry-ffi", @@ -6206,7 +6206,7 @@ dependencies = [ [[package]] name = "perry-ext-parcel-watcher" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "notify", "perry-ffi", @@ -6218,7 +6218,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "printpdf", @@ -6226,7 +6226,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "sqlx", @@ -6235,7 +6235,7 @@ dependencies = [ [[package]] name = "perry-ext-qs" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-runtime", @@ -6244,7 +6244,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "governor", "perry-ffi", @@ -6252,7 +6252,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "fast_image_resize", "image", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "perry-ext-streams" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "lazy_static", "perry-ffi", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "perry-ext-typescript" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-ffi", @@ -6292,7 +6292,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-runtime", @@ -6301,7 +6301,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "uuid", @@ -6309,7 +6309,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-validation", @@ -6318,7 +6318,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "futures-util", "lazy_static", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "brotli", "flate2", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "dashmap 6.2.1", "once_cell", @@ -6351,7 +6351,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-api-manifest", @@ -6372,11 +6372,11 @@ dependencies = [ [[package]] name = "perry-native-registration" -version = "0.5.1556" +version = "0.5.1557" [[package]] name = "perry-parser" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-diagnostics", @@ -6390,7 +6390,7 @@ dependencies = [ [[package]] name = "perry-perex" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perex", "regex", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "ahash", "anyhow", @@ -6458,14 +6458,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6560,14 +6560,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "perry-hir", @@ -6576,7 +6576,7 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "perry-ffi", "perry-ui-model", @@ -6584,7 +6584,7 @@ dependencies = [ [[package]] name = "perry-ui-android" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "itoa", @@ -6602,7 +6602,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "rand 0.10.2", "serde", @@ -6612,7 +6612,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "cairo-rs 0.22.9", @@ -6635,7 +6635,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "block2", @@ -6652,7 +6652,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "block2", @@ -6669,7 +6669,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1556" +version = "0.5.1557" [[package]] name = "perry-ui-test" @@ -6680,11 +6680,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1556" +version = "0.5.1557" [[package]] name = "perry-ui-tvos" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "block2", @@ -6701,7 +6701,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "block2", @@ -6718,7 +6718,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "block2", "libc", @@ -6732,7 +6732,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "libc", @@ -6751,7 +6751,7 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "base64 0.22.1", "libc", @@ -6764,7 +6764,7 @@ dependencies = [ [[package]] name = "perry-updater" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "anyhow", "base64 0.22.1", @@ -6780,7 +6780,7 @@ dependencies = [ [[package]] name = "perry-validation" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "idna", "regex", @@ -6790,7 +6790,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1556" +version = "0.5.1557" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index 5d704de8b7..9b3dfc8fa6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,7 +338,7 @@ codegen-units = 1 codegen-units = 1 [workspace.package] -version = "0.5.1556" +version = "0.5.1557" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry" diff --git a/changelog.d/10185-element-shape-fields-random.md b/changelog.d/10185-element-shape-fields-random.md new file mode 100644 index 0000000000..07b0533c87 --- /dev/null +++ b/changelog.d/10185-element-shape-fields-random.md @@ -0,0 +1,158 @@ +Serve the JSON access benchmark's `fields` and `random` shapes from the +element-shape versioned loop clone (#7480 / #5093 / #10171). Stacked on +#10171, which made the clone fire for `JSON.parse`'d record arrays at all. + +#10171 took the benchmark's `repeat` and `sequential` modes to parity. The two +remaining modes got nothing, for three separate reasons — and because a loop is +admitted as a whole, any one of them cost the entire clone: + +```ts +// random: a loop-carried index +cursor = (cursor * 17 + 7) % length; +const index = cursor; +sum += rows[index].id; + +// fields: three accumulator statements, one string read, one boolean ternary +const index = i % length; +sum += rows[index].id; +sum += rows[index].name.length; +sum += rows[index].active ? 1 : 0; +``` + +* **A loop-carried index** (`stmt/element_shape_carried.rs`). The recurrence is + folded to one affine pair `(a, b)` and evaluated as `srem i64` — the generic + `%` is an `frem`, which on aarch64 is a libm call, and a call inside this + clone does not slow it down, it DELETES it (#7690). Three obligations, all + discharged once in the preheader: the modulus materializes as an i32 in + `1..=i32::MAX` (so `m <= length` puts every index in bounds with no per-read + test), the carried binding's ENTRY value materializes as a non-negative + integral i32, and `a`/`b` are required non-negative so the dividend cannot go + negative — JS `%` returns a *negative* remainder for a negative dividend, + which is an out-of-bounds subscript rather than a slow path. + + The matcher also tracks the largest magnitude any sub-expression of the + recurrence can reach and declines above 2^53. "Fits i64" is necessary and NOT + sufficient: JavaScript evaluates the recurrence in doubles, and + `cursor * 1e9 + 1` fits i64 comfortably while losing its low bits in the f64 + the program actually runs. The bound is tracked per NODE rather than taken + from the folded pair, because folding cancels: `c * 1000 - c * 999` is + `1 * c`, but JavaScript still evaluates both thousand-fold products. + +* **Where the carried write-back goes is a correctness question, not a + scheduling one.** The residual side exit resumes the CURRENT iteration in the + slow clone, which re-runs the whole body — the recurrence included. So the + commit to the real binding is the LAST statement of the iteration: a + mid-iteration exit then leaves the binding holding that iteration's entry + value and the slow clone advances it exactly once. A write-back at the update + site double-applies it, and every later index is silently a different — still + in-bounds, still a valid record — one. + +* **K accumulator statements**, folded into one for the fast clone + (`acc = a; acc = acc + b` ≡ `acc = (a) + b`: the same operations on the same + operands in the same order, so the float result is bit-identical). Same + reason as above — the whole iteration must commit once, past every exit it + can take, or a tag test that fails on the third read leaves the first two + already applied when the slow clone re-runs it. Statements 2..K must read the + accumulator as the LEFT operand and must not read it in the addend, or the + substitution would change which value the addend sees. + +* **`arr[i].prop.length` and `arr[i].prop ? A : B`** + (`expr/element_shape_reads.rs`). The preheader proves which inline slot holds + a property; it proves nothing about what is in it. So each read tag-tests the + loaded word and side-exits when the answer is not the representation it is + about to assume: both string forms for `.length` (heap `utf16_len`, and the + SSO length byte — the same decode the runtime's own `.length` arms use), and + the two boolean singletons by exact NaN-box bit pattern for the ternary. + JS truthiness of `0` / `""` / `null` / an object is a runtime question the + clone does not guess, and a non-constant ternary arm is not admitted at all. + +* **One residual check per iteration, not one per read.** Three reads of + `rows[index]` were three element loads and three header/ShapeId checks. The + body's leading virtual binding now emits the deref and the residual once and + parks the masked handle in an entry alloca. That is also what makes the + multi-read side exit correct: every residual exit now precedes every store. + +* **The property denylist is arm-specific now.** The class-keyed arm keeps the + full list — its read bakes in a compile-time packed slot index while the + surrounding lowering may route the name elsewhere. The shape-keyed arm bakes + in nothing: it asks the runtime for that exact key's inline slot in that exact + ordinary ShapeId and declines on `-1`, and the residual pins + `obj_type == GC_TYPE_OBJECT` with no per-object descriptors — so every + receiver whose builtin branch could answer a name differently (a function's + `name`, an array's `length`, a Map's `size`) is already excluded, and for a + plain record an own data property SHADOWS the prototype name it collides + with. Only `__proto__` stays denied, and not because of JavaScript (node + gives `JSON.parse('{"__proto__":1}')` an own data property and reads `1` back + from it): Perry's generic property path may special-case the name ahead of + own-property lookup, and the clone must agree with the path it is a clone of + wherever the two differ. The full list would otherwise have cost the + benchmark its `fields` mode outright, for a field literally called `name`. + +**Measured** (ns/iter, 1M iterations, 5 interleaved rounds, best of 5, +`PERRY_NO_AUTO_OPTIMIZE=1`, on a loaded shared host; `new` and `base` are both +built from this worktree, `base` at #10171's head `2b77e7d4fe`): + +| cell | base (#10171) | new | node 26.5.1 | bun 1.3.14 | new / best | +|---|---:|---:|---:|---:|---:| +| 16k `repeat` | 4.21 | 4.17 | 3.56 | 5.23 | 1.171 | +| 16k `sequential` | 4.17 | 4.30 | 4.57 | 6.25 | 0.941 | +| 16k `random` | 15.48 | **4.97** | 7.16 | 9.96 | **0.694** | +| 16k `fields` | 25.59 | **6.05** | 5.88 | 9.77 | **1.029** | +| 1m `repeat` | 4.15 | 4.17 | 3.19 | 5.05 | 1.307 | +| 1m `sequential` | 4.26 | 4.20 | 9.42 | 7.96 | 0.528 | +| 1m `random` | 16.00 | **5.32** | 11.71 | 11.17 | **0.476** | +| 1m `fields` | 28.14 | **6.17** | 12.70 | 14.91 | **0.486** | +| 20m `repeat` | 4.12 | 4.11 | 3.50 | 5.24 | 1.174 | +| 20m `sequential` | 4.20 | 4.19 | 6.98 | 8.51 | 0.600 | +| 20m `random` | 27.53 | **5.63** | 9.31 | 10.56 | **0.605** | +| 20m `fields` | 27.05 | **6.06** | 10.98 | 14.81 | **0.552** | + +The two targeted shapes move 3.0x-4.9x and land below the better of node and +bun on five of their six cells. The sixth, 16k `fields`, is the one that does +not reach parity: 6.05 against node's 5.88 (1.029x; 6.08 vs 5.83 on a +separate 12-round re-measure of just that pair). +That is the smallest fixture, where the whole array is in L1 and the guard +overhead is the only thing left — node's own `fields` is just 1.29x its +`sequential` there, because its inline caches have type feedback proving `name` +is a string and `active` a boolean, while this clone tag-tests both on every +read. Paying that test is what lets it side-exit instead of deoptimize, so it +is a floor of the design rather than a missing peephole. Reported rather than +rounded off. + +`repeat` and `sequential`, which #10171 already served, are unchanged within +noise in both wall clock and instruction count — `sequential`'s single read has +nothing to share, and `repeat`'s constant index has no leading binding to hang +the prologue off. + +Instructions retired per iteration on the 16k fixture (2M iterations minus a +0-iteration run, `/usr/bin/time -l`): + +| shape | base | new | +|---|---:|---:| +| random | 207.6 | 29.1 | +| fields | 466.4 | 47.0 | +| sequential | 25.0 | 25.6 | +| repeat | 16.0 | 16.1 | + +`random` lands one recurrence above `sequential`'s 25, and `fields` two extra +guarded reads plus a string decode and a select above it — which is what the +shared residual buys, and the shape the numbers have to have if the clone is +really being entered rather than merely emitted (#10171's own lesson: a clone +can be `cond_br`-entered and never executed while every IR-census assertion +passes, and only the wall clock notices). + +Validation: `test-files/test_gap_json_record_loop_clone.ts` gains 40 cases — +the recurrence with and without its alias, the carried value read after the +loop, negative and fractional entry values, a zero and an over-long modulus, a +multiplier past the exact-double range, a mid-loop side exit whose sum AND +final cursor both observe the write-back protocol, SSO and heap `name`s, a +non-string and a null `name`, five non-boolean `active` values, a two-statement +fold whose second read side-exits, and records with own `name`/`length`/`size` +properties — all byte-identical to `node --experimental-strip-types` 26.5.1. +70 codegen IR-census tests (`element_shape_fields_random_tests.rs` and its +siblings), every positive paired with a sabotage case asserting the clone is +ABSENT. Seeded GC stress on the gap binary +(`PERRY_GC_SCHEDULE_SEED=1..4 PERRY_GC_SCHEDULE_RATE=0.2 +PERRY_GC_PROTECT_FROMSPACE=1 PERRY_GC_PROTECT_FROMSPACE_DEPTH=32`): output +identical across all four seeds and to node, with 3–11 copying minors and +~14.4k moved objects per run, so the instrument's subject was live. diff --git a/changelog.d/10189-class-field-get-one-exit.md b/changelog.d/10189-class-field-get-one-exit.md new file mode 100644 index 0000000000..7eb9ba9a14 --- /dev/null +++ b/changelog.d/10189-class-field-get-one-exit.md @@ -0,0 +1,10 @@ +Collapse the monomorphic class-field GET tower to one runtime exit. The #5093 +inline precheck and the fast slot load are unchanged; the guard call, the +nullish-throw diamond and the by-name fallback arms are replaced by a single +call to the existing `js_class_field_get_ic`, which performs the same guard, +load, fallback and throw. Per site that is 88 → 47 IR instructions, 3 → 1 +runtime call sites and 6 → 1 RS4GC relocation reloads; the fast path no longer +crosses a statepoint, so it sheds the two relocation reloads it used to carry. +On @babel/parser `.text` shrinks 7.1 % and `.perry_gcmap` 2.0 %, with runtime +instructions and RSS unchanged and `interp.ts` 13 % faster; typed-feedback +guard counters are identical per site. diff --git a/changelog.d/10196-generic-get-two-exits.md b/changelog.d/10196-generic-get-two-exits.md new file mode 100644 index 0000000000..9ba6cff0dc --- /dev/null +++ b/changelog.d/10196-generic-get-two-exits.md @@ -0,0 +1,12 @@ +Collapse the generic (untyped) property-get inline cache from a 33-block diamond +with six runtime call sites to the inline hit path plus the polymorphic ways and +two out-of-line exits: `js_object_get_field_ic_nonptr` for non-pointer receivers +(SSO, class refs, nullish throw, primitives) and `js_object_get_field_ic_slow` +for a pointer receiver that failed a guard (overflow slot, Array-subclass +named-prefix proofs, miss and prime). Per site: 191 → 106 IR instructions, +6 → 2 call sites. The packed-MRU hit path, the ways and every cache decision are +unchanged; the field address is emitted as a typed `gep` so instruction selection +keeps the scaled addressing mode. On @babel/parser `.text` shrinks 14.4 % and +`.perry_gcmap` 11.7 %; total benchmark instructions drop 0.5 % (`interp.ts` +−1.5 %) with RSS unchanged. A new gap test exercises every arm that moved out of +line against Node. diff --git a/crates/perry-codegen/src/codegen/argument_shape_clone_tests.rs b/crates/perry-codegen/src/codegen/argument_shape_clone_tests.rs index d2bd3ad63f..92d05f7a9b 100644 --- a/crates/perry-codegen/src/codegen/argument_shape_clone_tests.rs +++ b/crates/perry-codegen/src/codegen/argument_shape_clone_tests.rs @@ -210,13 +210,20 @@ fn guarded_call_routes_to_shadow_rooted_direct_field_clone() { clone.contains("inttoptr i64") && clone.contains("getelementptr double"), "the clone must use direct declared-field addressing:\n{clone}" ); + // One-exit class-field GET: the tower's miss arm is a single + // `js_class_field_get_ic` call whose body IS the guard, so the IC is now a + // witness of "a field IC diamond was rebuilt" and must be excluded here + // too -- otherwise this negative assertion silently stops covering the + // shape it was written for. assert!( !clone.contains("js_typed_feedback_class_field_get_guard") + && !clone.contains("js_class_field_get_ic") && !clone.contains("shape_descriptor_by_id"), "the clone fast body must not rebuild the field IC diamond:\n{clone}" ); assert!( - generic.contains("js_typed_feedback_class_field_get_guard") + generic.contains("js_class_field_get_ic") + || generic.contains("js_typed_feedback_class_field_get_guard") || generic.contains("js_object_get_field"), "the generic fallback must retain guarded field semantics:\n{generic}" ); @@ -264,6 +271,7 @@ fn unannotated_parameter_uses_the_runtime_validated_class_overlay() { ); assert!( !clone.contains("js_typed_feedback_class_field_get_guard") + && !clone.contains("js_class_field_get_ic") && !clone.contains("shape_descriptor_by_id"), "the unannotated clone must not rebuild the field IC diamond:\n{clone}" ); diff --git a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs index 03bbccdb48..89d929b797 100644 --- a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs +++ b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs @@ -347,7 +347,9 @@ fn additive_property_callback_gets_a_cold_deopting_private_body() { ); assert!( special.contains("pic.miss.call") - && special.contains("js_object_get_field_ic_miss") + // T1 renamed the property-GET slow path this cold arm reaches + // (`js_object_get_field_ic_miss_packed` -> the tower's object exit). + && special.contains("js_object_get_field_ic_slow") && special.contains("guarded_add.dynamic") && special.contains("versioned_callback.deopt.mark"), "both observable cold arms must poison the loop before fallback:\n{special}" diff --git a/crates/perry-codegen/src/eh_mode.rs b/crates/perry-codegen/src/eh_mode.rs index 0c436443d0..d694ff75a3 100644 --- a/crates/perry-codegen/src/eh_mode.rs +++ b/crates/perry-codegen/src/eh_mode.rs @@ -105,9 +105,19 @@ mod tests { #[test] fn cold_property_miss_is_still_throwing() { - let name = "js_object_get_field_ic_miss_packed"; - assert_eq!(crate::module::helper_decl_attrs(name), " cold"); - assert!(!callee_is_nothrow(name)); + for name in [ + "js_object_get_field_ic_miss_packed", + // T1: the generic-get tower's two slow exits carry the same `cold` + // placement hint — and the same ability to throw, now including + // the nullish-receiver TypeError the emitted arm used to raise + // inline. A `cold` helper that lost its invoke edge would silently + // swallow every `Cannot read properties of undefined`. + "js_object_get_field_ic_slow", + "js_object_get_field_ic_nonptr", + ] { + assert_eq!(crate::module::helper_decl_attrs(name), " cold", "{name}"); + assert!(!callee_is_nothrow(name), "{name}"); + } assert!(!callee_is_nothrow("js_object_get_field_ic_miss")); assert!(!callee_is_nothrow("unknown_runtime_helper")); } diff --git a/crates/perry-codegen/src/expr/array_callback_shape_tests.rs b/crates/perry-codegen/src/expr/array_callback_shape_tests.rs index 5258bb195b..13ad7f81ca 100644 --- a/crates/perry-codegen/src/expr/array_callback_shape_tests.rs +++ b/crates/perry-codegen/src/expr/array_callback_shape_tests.rs @@ -150,8 +150,13 @@ fn callback_body(ir: &str) -> &str { fn inline_array_callback_field_read_has_no_shape_guard() { let ir = emit(&module_with_callback(false)); let callback = callback_body(&ir); + // `js_class_field_get_ic` is the one-exit tower's miss arm (guard + slot + // load + by-name fallback in one call), so it is the current witness that a + // guarded read was emitted; without it this negative assertion would pass + // on a callback that is still fully guarded. assert!( !callback.contains("js_typed_feedback_class_field_get_guard") + && !callback.contains("js_class_field_get_ic") && !callback.contains("js_object_get_field_by_name_f64"), "the proven element parameter must use a direct fixed-offset load:\n{callback}" ); @@ -166,7 +171,8 @@ fn callback_source_array_alias_keeps_the_shape_guard() { let ir = emit(&module_with_callback(true)); let callback = callback_body(&ir); assert!( - callback.contains("js_typed_feedback_class_field_get_guard") + callback.contains("js_class_field_get_ic") + || callback.contains("js_typed_feedback_class_field_get_guard") || callback.contains("js_object_get_field_by_name_f64"), "declaring the source-array argument must deny the cross-boundary fact:\n{callback}" ); diff --git a/crates/perry-codegen/src/expr/binary.rs b/crates/perry-codegen/src/expr/binary.rs index 04be6b3799..ebeac06c0f 100644 --- a/crates/perry-codegen/src/expr/binary.rs +++ b/crates/perry-codegen/src/expr/binary.rs @@ -504,6 +504,17 @@ fn reduction_add_is_reassociable(ctx: &FnCtx<'_>, left: &Expr, right: &Expr) -> } fn lower_arithmetic_operand(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<(String, bool)> { + // #10185: the element-shape fast clone's two NON-numeric reads — + // `arr[i].name.length` and `arr[i].active ? 1 : 0`. Both are numbers by the + // time they get here, each proven from the loaded word's own NaN-box tag + // with a side exit to the slow clone on a miss, so the `true` suppresses + // the residual `js_number_coerce` below: that call would not slow the clone + // down, it would delete it. Outside a clone the fact vector is empty and + // this is a no-op. See `expr::element_shape_reads`. + if let Some(value) = crate::expr::element_shape_reads::try_lower_cloned_read_operand(ctx, expr)? + { + return Ok((value, true)); + } // A stable-packed numeric clone has a stronger fact than the generic // untyped-local typed-array probe below: its preheader scanned the exact // range, and its emitted-IR gate proved that no call can invalidate that diff --git a/crates/perry-codegen/src/expr/class_field_get_shape_tests.rs b/crates/perry-codegen/src/expr/class_field_get_shape_tests.rs new file mode 100644 index 0000000000..d07a664bae --- /dev/null +++ b/crates/perry-codegen/src/expr/class_field_get_shape_tests.rs @@ -0,0 +1,310 @@ +//! The monomorphic class-field GET tower has ONE exit. +//! +//! The tower in [`super::property_get`] used to emit four runtime call sites +//! per `this.field` / `obj.field` read — `js_typed_feedback_class_field_get_ +//! guard` when the #5093 inline pre-check missed, `js_throw_type_error_ +//! property_access` for a nullish receiver, and one or two +//! `js_object_get_field_by_name_f64` by-name lookups — spread over +//! `class_field_get.{fast,fallback,merge,throw_nullish,fallback_lookup}`. At +//! ~74 pre-RS4GC instructions and 13,636 sites in @babel/parser that arm alone +//! was 27 % of the module's IR, and because every call is a statepoint the +//! `.perry_gcmap` grew with it. +//! +//! Everything behind the pre-check now lives in `js_class_field_get_ic`, the +//! runtime helper the #5391 path-2 full outline already called. What must hold, +//! and why a label-presence check cannot show it: +//! +//! 1. **The inline hit path is still there and still REACHED.** A one-exit +//! tower that also lost its pre-check would be a pure pessimisation, and +//! every assertion below about "one call" would still pass. So: a `cond_br` +//! out of `class_field_inline.deref` INTO `class_field_get.fast`, and a +//! `load double` inside that block. (CLAUDE.md, "a gate must assert its +//! subject was live".) +//! 2. **The miss arm is exactly one call.** Counted, not merely "the IC is +//! mentioned": a regression that re-adds the by-name fallback beside the IC +//! would keep every positive assertion true. +//! 3. **The retired symbols are not CALLED at this site.** Asserted as +//! `call @name(`, never as a bare substring — all three are still +//! declared in the module (and still emitted by other towers), so a +//! substring test is satisfied by the `declare` line alone and can never +//! fail. That exact trap is why the older assertions in +//! `tests/typed_feedback.rs` were rewritten rather than deleted. +//! 4. **Both representations.** A `number` field (`require_raw_f64 = 1`) and an +//! `any` field (boxed) take the same shape; the raw-f64 site must not grow a +//! second by-name call back. + +use crate::compile_module; +use perry_hir::types::Type; +use perry_hir::{Class, ClassField, Expr, Function, Module, ModuleInitKind, Param, Stmt}; + +const PARAM_ID: u32 = 3; + +/// Blocks the tower is allowed to create, in the order it creates them. +const TOWER_BLOCKS: [&str; 4] = [ + "class_field_inline.deref", + "class_field_inline.guardcall", + "class_field_get.fast", + "class_field_get.merge", +]; + +/// Blocks the four-exit tower created that must no longer exist. +const RETIRED_BLOCKS: [&str; 3] = [ + "class_field_get.fallback", + "class_field_get.throw_nullish", + "class_field_get.fallback_lookup", +]; + +/// Call FORMS (not bare names) for the runtime entries the tower stopped +/// emitting. Each is still declared in every module and still emitted by other +/// lowerings, so only the call form can distinguish "not called here". +const RETIRED_CALLS: [&str; 3] = [ + "call i32 @js_typed_feedback_class_field_get_guard(", + "call void @js_throw_type_error_property_access(", + "call double @js_object_get_field_by_name_f64(", +]; + +fn field(name: &str, ty: Type) -> ClassField { + ClassField { + name: name.to_string(), + key_expr: None, + ty, + init: None, + is_private: false, + is_readonly: false, + decorators: Vec::new(), + } +} + +fn point_class(field_ty: Type) -> Class { + Class { + id: 101, + name: "Point".to_string(), + type_params: Vec::new(), + extends: None, + extends_name: None, + native_extends: None, + extends_expr: None, + heritage_lexically_shadowed: false, + fields: vec![field("x", field_ty)], + constructor: None, + methods: Vec::new(), + getters: Vec::new(), + setters: Vec::new(), + static_accessor_names: Vec::new(), + static_accessor_fn_ids: Vec::new(), + computed_members: Vec::new(), + static_fields: Vec::new(), + static_methods: Vec::new(), + decorators: Vec::new(), + is_exported: false, + aliases: Vec::new(), + is_nested: false, + alloc_width_hint: 0, + specialized_from: None, + } +} + +/// `function probe(p: Point) { return p.x }` — a bare `Named` parameter, which +/// is what routes to the generic `class_field_get.*` tower (#8033: the +/// numeric-specific `class_field_get_number.*` path needs a +/// `stable_local_type_proof` a parameter has not got). +fn probe_module(field_ty: Type) -> Module { + let mut m = Module::new("class_field_get_shape.ts"); + m.classes = vec![point_class(field_ty)]; + m.functions = vec![Function { + id: 1, + name: "probe".to_string(), + type_params: Vec::new(), + params: vec![Param { + id: PARAM_ID, + name: "p".to_string(), + ty: Type::Named("Point".to_string()), + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }], + return_type: Type::Any, + body: vec![Stmt::Return(Some(Expr::PropertyGet { + byte_offset: 0, + object: Box::new(Expr::LocalGet(PARAM_ID)), + property: "x".to_string(), + }))], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }]; + m.init_kind = ModuleInitKind::Eager; + m +} + +fn ir(field_ty: Type) -> String { + String::from_utf8( + compile_module( + &probe_module(field_ty), + super::class_field_barrier_tests::ir_opts(), + ) + .expect("module compiles"), + ) + .expect("LLVM IR should be UTF-8") +} + +/// Body of the first block whose label line starts with `prefix`. Block labels +/// carry per-function numeric suffixes (`.fast.6`), so callers pass the stable +/// prefix; a label line is line-initial and ends in `:`, while branches and +/// phis that merely mention it are indented. +fn block_body<'a>(ir: &'a str, prefix: &str) -> Option<&'a str> { + let needle = format!("\n{prefix}"); + let mut from = 0; + while let Some(rel) = ir[from..].find(&needle) { + let label_start = from + rel + 1; + let line_end = label_start + ir[label_start..].find('\n')?; + if ir[label_start..line_end].ends_with(':') { + let rest = &ir[line_end + 1..]; + let end = match (rest.find("\n\n"), rest.find("\n}")) { + (Some(a), Some(b)) => a.min(b), + (a, b) => a.or(b).unwrap_or(rest.len()), + }; + return Some(&rest[..end]); + } + from = line_end; + } + None +} + +/// The `prefix.N:` label as rendered, e.g. `class_field_get.fast.6`. +fn block_label(ir: &str, prefix: &str) -> Option { + let needle = format!("\n{prefix}"); + let mut from = 0; + while let Some(rel) = ir[from..].find(&needle) { + let label_start = from + rel + 1; + let line_end = label_start + ir[label_start..].find('\n')?; + let line = &ir[label_start..line_end]; + if line.ends_with(':') { + return Some(line.trim_end_matches(':').to_string()); + } + from = line_end; + } + None +} + +fn assert_one_exit(field_ty: Type, what: &str) { + let ir = ir(field_ty); + + // (1) the pre-check exists and BRANCHES INTO the fast slot load. + let deref = block_body(&ir, "class_field_inline.deref") + .unwrap_or_else(|| panic!("{what}: no class_field_inline.deref block:\n{ir}")); + let fast_label = block_label(&ir, "class_field_get.fast") + .unwrap_or_else(|| panic!("{what}: no class_field_get.fast block:\n{ir}")); + assert!( + deref.contains("br i1 ") && deref.contains(&format!("label %{fast_label}")), + "{what}: class_field_inline.deref does not branch into {fast_label}; \ + the inline hit path is dead:\n{deref}" + ); + let fast = block_body(&ir, "class_field_get.fast") + .unwrap_or_else(|| panic!("{what}: no class_field_get.fast body:\n{ir}")); + assert!( + fast.contains("load double"), + "{what}: the fast block no longer loads the slot:\n{fast}" + ); + assert!( + !fast.contains("call "), + "{what}: the fast block must stay call-free:\n{fast}" + ); + + // (2) the miss arm is EXACTLY one call. + let guardcall = block_body(&ir, "class_field_inline.guardcall") + .unwrap_or_else(|| panic!("{what}: no class_field_inline.guardcall block:\n{ir}")); + let calls = guardcall.matches("call ").count(); + assert_eq!( + calls, 1, + "{what}: the miss arm must be ONE call, found {calls}:\n{guardcall}" + ); + assert!( + guardcall.contains("call double @js_class_field_get_ic("), + "{what}: the miss arm's one call is not js_class_field_get_ic:\n{guardcall}" + ); + + // (3) the retired blocks and the retired CALL FORMS are gone. + for block in RETIRED_BLOCKS { + assert!( + block_body(&ir, block).is_none(), + "{what}: retired block {block} is still emitted:\n{ir}" + ); + } + for call in RETIRED_CALLS { + assert!( + !ir.contains(call), + "{what}: retired call form `{call}` is still emitted:\n{ir}" + ); + } + + // (4) the merge is a two-way phi (fast value, IC value) and nothing else. + let merge = block_body(&ir, "class_field_get.merge") + .unwrap_or_else(|| panic!("{what}: no class_field_get.merge block:\n{ir}")); + let phis: Vec<&str> = merge.lines().filter(|l| l.contains("phi double")).collect(); + assert_eq!(phis.len(), 1, "{what}: merge is not a single phi:\n{merge}"); + assert_eq!( + phis[0].matches('[').count(), + 2, + "{what}: merge phi does not have exactly two incoming edges (fast, IC):\n{}", + phis[0] + ); + + // (5) the tower creates exactly four blocks. + for block in TOWER_BLOCKS { + assert!( + block_body(&ir, block).is_some(), + "{what}: tower block {block} missing:\n{ir}" + ); + } +} + +#[test] +fn boxed_class_field_get_has_one_exit() { + assert_one_exit(Type::Any, "any-typed field"); +} + +#[test] +fn raw_f64_class_field_get_has_one_exit() { + // `number` makes `requires_raw_f64` true, which used to emit a SECOND + // by-name fallback record/call pair. The IC takes `require_raw_f64` as an + // argument instead, so the site stays at one call. + assert_one_exit(Type::Number, "number-typed field"); +} + +#[test] +fn the_ic_call_carries_the_guard_operands() { + // The helper runs the same guard the inline miss arm used to call, so it + // must receive the same seven operands in the same order — site id, + // receiver, expected class id, expected shape id, key, field index, + // require_raw_f64. A dropped operand compiles and silently guards on the + // wrong thing. + let ir = ir(Type::Number); + let guardcall = block_body(&ir, "class_field_inline.guardcall") + .expect("guardcall block") + .to_string(); + let line = guardcall + .lines() + .find(|l| l.contains("@js_class_field_get_ic(")) + .expect("the IC call line"); + let args = line + .rsplit_once("@js_class_field_get_ic(") + .expect("call args") + .1; + let args = args.rsplit_once(')').expect("closing paren").0; + let tys: Vec<&str> = args + .split(", ") + .map(|a| a.split_whitespace().next().unwrap_or("")) + .collect(); + assert_eq!( + tys, + vec!["i64", "double", "i32", "i32", "i64", "i32", "i32"], + "IC call signature drifted from the guard's operand list:\n{line}" + ); +} diff --git a/crates/perry-codegen/src/expr/element_shape_guard.rs b/crates/perry-codegen/src/expr/element_shape_guard.rs index d2ec6ce903..a8565b21b3 100644 --- a/crates/perry-codegen/src/expr/element_shape_guard.rs +++ b/crates/perry-codegen/src/expr/element_shape_guard.rs @@ -538,13 +538,137 @@ pub(crate) fn emit_element_shape_loop_preheader_check( }) } -/// Emit one `arr[i].field` read inside the fast clone: bare element load, -/// an optional residual per-element check, then a bare raw-f64 slot load. +/// Materialize this iteration's element index as an i32, or report that the +/// clone cannot index at all. /// -/// `idx_i32` must be an index the preheader discharged a bounds obligation for -/// (see [`ElementShapeIndexBound`]), and `fact` must be the fact the preheader -/// installed for this array. -pub(crate) fn emit_element_shape_field_load( +/// One arm per [`super::ElementShapeIndex`] spelling, and each reads exactly +/// the storage whose bounds obligation the preheader discharged +/// ([`ElementShapeIndexBound`]). `None` means the counter's canonical i32 slot +/// the matcher required has gone missing — that costs the read its fast +/// lowering, never correctness, and the matcher, the fact lookup and this must +/// all agree (`ElementShapeIndex::needs_counter_i32_slot`). +pub(crate) fn emit_element_shape_index( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, +) -> Option { + match &fact.index { + super::ElementShapeIndex::Counter => { + let slot = ctx.i32_counter_slots.get(&fact.index_local_id).cloned()?; + Some(ctx.block().load(I32, &slot)) + } + super::ElementShapeIndex::Constant(k) => Some(k.to_string()), + // The derived `const d = j % m` binding's own slot, written by the + // `Let` arm in `stmt/let_stmt.rs` earlier in this same iteration. + super::ElementShapeIndex::DerivedMod { slot, .. } => { + let slot = slot.clone(); + Some(ctx.block().load(I32, &slot)) + } + // #10185: the carried recurrence's private i32 slot, written by the + // body's first statement earlier in this same iteration. The REAL + // binding slot is deliberately NOT read here — it is one iteration + // behind until the trailing write-back commits, which is exactly what + // makes a mid-iteration side exit correct. + super::ElementShapeIndex::Carried(carried) => { + let slot = carried.slot.clone(); + Some(ctx.block().load(I32, &slot)) + } + } +} + +/// The masked heap handle of `arr[idx]` for THIS iteration, with the residual +/// per-element facts discharged. +/// +/// Two sources. When the matcher installed an [`super::ElementPrefetch`] the +/// body's leading virtual binding already did the deref and the residual check +/// for the whole iteration, so this is one load of an entry alloca; otherwise +/// (#10123's original shape) the deref and the check are emitted here, once per +/// read. +pub(crate) fn emit_element_shape_element_handle( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, + idx_i32: &str, +) -> String { + if let Some(prefetch) = &fact.elem_prefetch { + let slot = prefetch.handle_slot.clone(); + return ctx.block().load(I64, &slot); + } + emit_element_deref_with_residual(ctx, fact, idx_i32) +} + +/// Bare element load plus the residual per-OBJECT facts the array-level +/// invariant deliberately does not cover. Returns the masked handle. +/// +/// Emits nothing but loads, ALU and one `cond_br` — call-free by construction, +/// which is the whole revocation argument (see the module docs). +pub(crate) fn emit_element_deref_with_residual( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, + idx_i32: &str, +) -> String { + let blk = ctx.block(); + // The element-shape invariant proved every slot in the verified prefix + // is a POINTER_TAG object of the guarded identity, so the unbox needs + // no tag test and no handle-band test — the two checks that make up the + // element-read tier. + let idx64 = blk.sext(I32, idx_i32, I64); + let slot_ptr = blk.gep(I64, &fact.elements_base, &[(I64, &idx64)]); + let elem_bits = blk.load(I64, &slot_ptr); + let elem_handle = blk.and(I64, &elem_bits, crate::nanbox::POINTER_MASK_I64); + + if fact.statically_layout_proven { + return elem_handle; + } + let elem_ptr = blk.inttoptr(I64, &elem_handle); + let load_idx = ctx.new_block("element_shape.load"); + let load_label = ctx.block_label(load_idx); + let blk = ctx.block(); + + // Residual per-OBJECT facts (see the module docs for why they cannot come + // from the runtime array-level invariant). + let hdr_ptr = blk.gep(I8, &elem_ptr, &[(I64, "-8")]); + let hdr = blk.load(I32, &hdr_ptr); + let (mask, expect) = if fact.shape_keyed { + (ELEM_HEADER_SHAPE_MASK, ELEM_HEADER_SHAPE_EXPECT) + } else { + (ELEM_HEADER_MASK, ELEM_HEADER_EXPECT) + }; + let hdr_masked = blk.and(I32, &hdr, mask); + let hdr_ok = blk.icmp_eq(I32, &hdr_masked, expect); + + // #8113: the ShapeId moved from header offset 8 to 4. + let sid_ptr = blk.gep(I8, &elem_ptr, &[(I64, "4")]); + let shape_id = blk.load(I32, &sid_ptr); + let shape_ok = blk.icmp_eq(I32, &shape_id, &fact.expected_shape_id); + + let ok = blk.and(I1, &hdr_ok, &shape_ok); + // The side exit resumes the CURRENT iteration in the slow clone; no effect + // of this iteration has committed yet (#10185: with a prefetch this is the + // ONLY residual exit in the iteration, and it precedes every store). + blk.cond_br(&ok, &load_label, &fact.side_exit_label); + ctx.current_block = load_idx; + elem_handle +} + +/// #10185: the once-per-iteration element prologue. +/// +/// Emitted by the body's leading virtual binding (`stmt/let_stmt.rs` → +/// `stmt/element_shape_loop::lower_virtual_clone_binding`) once the index for +/// this iteration is in its slot. Parks the masked handle in an entry alloca so +/// every read in the iteration is a bare offset load. +pub(crate) fn emit_element_shape_prefetch( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, + idx_i32: &str, + handle_slot: &str, +) { + let handle = emit_element_deref_with_residual(ctx, fact, idx_i32); + ctx.block().store(I64, &handle, handle_slot); +} + +/// One tracked property's RAW slot word — a NaN-boxed `JSValue` in the +/// shape-keyed arm, a raw `double` in the class-keyed one. No tag test; the +/// caller adds the one its consumer needs. +pub(crate) fn emit_element_shape_slot_load( ctx: &mut FnCtx, fact: &super::ElementShapeLoopFact, idx_i32: &str, @@ -555,54 +679,27 @@ pub(crate) fn emit_element_shape_field_load( super::ElementShapeFieldSlot::Packed(index) => (I64, index.to_string()), super::ElementShapeFieldSlot::Runtime(reg) => (I64, reg.clone()), }; - - let elem_ptr = { - let blk = ctx.block(); - // The element-shape invariant proved every slot in the verified prefix - // is a POINTER_TAG object of the guarded identity, so the unbox needs - // no tag test and no handle-band test — the two checks that make up the - // element-read tier. - let idx64 = blk.sext(I32, idx_i32, I64); - let slot_ptr = blk.gep(I64, &fact.elements_base, &[(I64, &idx64)]); - let elem_bits = blk.load(I64, &slot_ptr); - let elem_handle = blk.and(I64, &elem_bits, crate::nanbox::POINTER_MASK_I64); - let elem_ptr = blk.inttoptr(I64, &elem_handle); - - if !fact.statically_layout_proven { - let load_idx = ctx.new_block("element_shape.load"); - let load_label = ctx.block_label(load_idx); - let blk = ctx.block(); - - // Residual per-OBJECT facts (see the module docs for why they - // cannot come from the runtime array-level invariant). - let hdr_ptr = blk.gep(I8, &elem_ptr, &[(I64, "-8")]); - let hdr = blk.load(I32, &hdr_ptr); - let (mask, expect) = if fact.shape_keyed { - (ELEM_HEADER_SHAPE_MASK, ELEM_HEADER_SHAPE_EXPECT) - } else { - (ELEM_HEADER_MASK, ELEM_HEADER_EXPECT) - }; - let hdr_masked = blk.and(I32, &hdr, mask); - let hdr_ok = blk.icmp_eq(I32, &hdr_masked, expect); - - // #8113: the ShapeId moved from header offset 8 to 4. - let sid_ptr = blk.gep(I8, &elem_ptr, &[(I64, "4")]); - let shape_id = blk.load(I32, &sid_ptr); - let shape_ok = blk.icmp_eq(I32, &shape_id, &fact.expected_shape_id); - - let ok = blk.and(I1, &hdr_ok, &shape_ok); - // One branch per access. The side exit resumes the CURRENT - // iteration in the slow clone; no effect has committed yet. - blk.cond_br(&ok, &load_label, &fact.side_exit_label); - ctx.current_block = load_idx; - } - elem_ptr - }; - + let elem_handle = emit_element_shape_element_handle(ctx, fact, idx_i32); let blk = ctx.block(); + let elem_ptr = blk.inttoptr(I64, &elem_handle); let fields_base = blk.gep(I8, &elem_ptr, &[(I64, &header_skip)]); let field_ptr = blk.gep(DOUBLE, &fields_base, &[(slot_ty, &slot_value)]); - let value = blk.load(DOUBLE, &field_ptr); + blk.load(DOUBLE, &field_ptr) +} + +/// Emit one `arr[i].field` read inside the fast clone: bare element load, +/// an optional residual per-element check, then a bare raw-f64 slot load. +/// +/// `idx_i32` must be an index the preheader discharged a bounds obligation for +/// (see [`ElementShapeIndexBound`]), and `fact` must be the fact the preheader +/// installed for this array. +pub(crate) fn emit_element_shape_field_load( + ctx: &mut FnCtx, + fact: &super::ElementShapeLoopFact, + idx_i32: &str, + field_slot: &super::ElementShapeFieldSlot, +) -> String { + let value = emit_element_shape_slot_load(ctx, fact, idx_i32, field_slot); // #10123: the shape-keyed arm's representation check. // diff --git a/crates/perry-codegen/src/expr/element_shape_reads.rs b/crates/perry-codegen/src/expr/element_shape_reads.rs new file mode 100644 index 0000000000..9d78d32174 --- /dev/null +++ b/crates/perry-codegen/src/expr/element_shape_reads.rs @@ -0,0 +1,326 @@ +//! #10185: the two NON-numeric reads the element-shape fast clone admits — +//! `arr[i].prop.length` on a string field and `arr[i].prop ? A : B` on a +//! boolean one. +//! +//! ## Why they need their own lowering at all +//! +//! Both are ordinary JavaScript that the generic lowering handles perfectly +//! well — with a call. `.length` on an untyped receiver reaches the property +//! diamond and, on a miss, `js_object_get_field_by_name_f64`; a ternary on an +//! untyped condition reaches `js_is_truthy`. Inside this clone a call is not a +//! slow path, it is a DELETED clone (`contains_gc_unsafe_call`, #7690), so the +//! benchmark's `fields` shape — +//! +//! ```text +//! sum += rows[index].id; +//! sum += rows[index].name.length; +//! sum += rows[index].active ? 1 : 0; +//! ``` +//! +//! got no clone at all: one unadmitted read in the body costs the whole loop. +//! +//! ## What each one proves before it reads +//! +//! The element-shape preheader proves WHICH inline slot holds `name` and +//! `active`; it proves nothing about what is in them. So each read tag-tests +//! the loaded word and side-exits to the slow clone when the answer is not the +//! representation it is about to assume — exactly the discipline #10123 +//! established for the Number case. A `name` that is a number, a `null` or an +//! object, and an `active` that is anything but `true`/`false`, are re-run in +//! the slow clone, where full JS semantics apply. +//! +//! **`.length`.** JS `.length` is UTF-16 code units. A heap string keeps that +//! count in `StringHeader::utf16_len`, the leading `u32` — the identical load +//! the inline `.length` fast path in `property_get/generic_dispatch.rs` emits. +//! An SSO immediate (`SHORT_STRING_TAG`, up to five bytes packed into the +//! NaN-box) keeps its BYTE length in bits 40..=47, and this reads it the same +//! way the runtime's own SSO `.length` arms do +//! (`string/char_ops.rs::string_property_get_miss`, +//! `property_get/generic_dispatch.rs`). That convention is byte length, not +//! code-unit length, so a non-ASCII SSO string reports its UTF-8 size — a +//! PRE-EXISTING Perry-wide answer, reproduced here deliberately: the clone must +//! agree with the path it is a clone of, and diverging from it would be a +//! miscompile even where the shared answer is itself wrong. +//! +//! **The ternary.** JS truthiness of an arbitrary value is a runtime question +//! (`""`, `0`, `NaN`, `null`, every object). The clone does not guess it: only +//! the two boolean singletons are admitted, by exact NaN-box bit pattern, and +//! everything else side-exits. The result is one `select` between two +//! compile-time constants. + +use anyhow::Result; +use perry_hir::Expr; + +use super::FnCtx; +use crate::types::{DOUBLE, I1, I32, I64}; + +/// `STRING_TAG >> 48` — a heap `StringHeader` pointer in the low 48 bits. +const STRING_TAG_TOP16: &str = crate::nanbox::STRING_TAG_TOP16_I64; +/// `SHORT_STRING_TAG >> 48` — an SSO immediate. +const SHORT_STRING_TAG_TOP16: &str = crate::nanbox::SHORT_STRING_TAG_TOP16_I64; +/// `SHORT_STRING_LEN_SHIFT` — the length byte sits at bits 40..=47. +const SHORT_STRING_LEN_SHIFT: &str = "40"; + +/// `TAG_TRUE` (`0x7FFC_0000_0000_0004`) as a decimal i64 literal. +const TAG_TRUE_I64: &str = "9222246136947933188"; +/// `TAG_FALSE` (`0x7FFC_0000_0000_0003`) as a decimal i64 literal. +const TAG_FALSE_I64: &str = "9222246136947933187"; + +/// Is `expr` a tracked element read the fast clone has a fact for? +/// +/// Only meaningful INSIDE the clone — outside one the fact vector is empty and +/// every answer is `None`. +fn tracked_element_read(ctx: &FnCtx<'_>, expr: &Expr) -> bool { + let Expr::PropertyGet { + object, property, .. + } = expr + else { + return false; + }; + super::element_shape_loop_fact_for_property_get(ctx, object, property).is_some() +} + +/// `.length` — returns the inner read. +pub(crate) fn cloned_string_length_read<'e>(ctx: &FnCtx<'_>, expr: &'e Expr) -> Option<&'e Expr> { + let Expr::PropertyGet { + object, property, .. + } = expr + else { + return None; + }; + if property != "length" { + return None; + } + tracked_element_read(ctx, object.as_ref()).then(|| object.as_ref()) +} + +/// ` ? : ` — returns the +/// condition's inner read and the two constants. +pub(crate) fn cloned_bool_select_read<'e>( + ctx: &FnCtx<'_>, + expr: &'e Expr, +) -> Option<(&'e Expr, i64, i64)> { + let Expr::Conditional { + condition, + then_expr, + else_expr, + } = expr + else { + return None; + }; + if !tracked_element_read(ctx, condition.as_ref()) { + return None; + } + Some(( + condition.as_ref(), + integer_arm(then_expr)?, + integer_arm(else_expr)?, + )) +} + +/// A ternary arm the clone may bake in: an integer-valued literal. +/// +/// Deliberately narrow. A non-constant arm would have to be lowered on both +/// sides of a branch inside a clone whose whole admission argument is that its +/// body is one straight line of pure reads. +pub(crate) fn integer_arm(expr: &Expr) -> Option { + match expr { + Expr::Integer(k) => Some(*k), + Expr::Number(n) if n.is_finite() && n.fract() == 0.0 && *n >= -1e15 && *n <= 1e15 => { + Some(*n as i64) + } + _ => None, + } +} + +/// Resolve one tracked read to (fact, slot, index) and emit the raw slot word. +fn emit_tracked_slot_word(ctx: &mut FnCtx<'_>, read: &Expr) -> Option { + let Expr::PropertyGet { + object, property, .. + } = read + else { + return None; + }; + let (fact, slot) = super::element_shape_loop_fact_for_property_get(ctx, object, property) + .map(|(fact, slot)| (fact.clone(), slot.clone()))?; + // A class-keyed clone's slot holds a RAW double, not a NaN-boxed value — + // there is no tag to test and these two readers would be reading a + // `double`'s bit pattern as a box. Both forms are shape-keyed-only. + if !fact.shape_keyed { + return None; + } + let idx_i32 = super::element_shape_guard::emit_element_shape_index(ctx, &fact)?; + Some(super::element_shape_guard::emit_element_shape_slot_load( + ctx, &fact, &idx_i32, &slot, + )) +} + +/// The side exit the tracked read's own fact names. +fn side_exit_label(ctx: &FnCtx<'_>, read: &Expr) -> Option { + let Expr::PropertyGet { + object, property, .. + } = read + else { + return None; + }; + super::element_shape_loop_fact_for_property_get(ctx, object, property) + .map(|(fact, _)| fact.side_exit_label.clone()) +} + +/// Emit `.length` as an f64. +pub(crate) fn lower_cloned_string_length( + ctx: &mut FnCtx<'_>, + read: &Expr, +) -> Result> { + let Some(exit) = side_exit_label(ctx, read) else { + return Ok(None); + }; + let Some(value) = emit_tracked_slot_word(ctx, read) else { + return Ok(None); + }; + + let bits = ctx.block().bitcast_double_to_i64(&value); + let top16 = ctx.block().lshr(I64, &bits, "48"); + let is_heap = ctx.block().icmp_eq(I64, &top16, STRING_TAG_TOP16); + let is_sso = ctx.block().icmp_eq(I64, &top16, SHORT_STRING_TAG_TOP16); + let is_string = ctx.block().or(I1, &is_heap, &is_sso); + + let dispatch_idx = ctx.new_block("element_shape.strlen"); + let heap_idx = ctx.new_block("element_shape.strlen.heap"); + let sso_idx = ctx.new_block("element_shape.strlen.sso"); + let done_idx = ctx.new_block("element_shape.strlen.done"); + let dispatch_label = ctx.block_label(dispatch_idx); + let heap_label = ctx.block_label(heap_idx); + let sso_label = ctx.block_label(sso_idx); + let done_label = ctx.block_label(done_idx); + + // A non-string `name` re-runs the whole iteration in the slow clone rather + // than being decoded as one. + ctx.block().cond_br(&is_string, &dispatch_label, &exit); + + ctx.current_block = dispatch_idx; + ctx.block().cond_br(&is_heap, &heap_label, &sso_label); + + // `StringHeader::utf16_len` is the leading `u32`; `safe_load_i32_from_ptr` + // keeps a sub-page handle off the load, and is itself call-free (an `icmp`, + // a `select` onto a zero-valued global, and the load). + ctx.current_block = heap_idx; + let heap_handle = ctx.block().and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + let heap_len_i32 = ctx.block().safe_load_i32_from_ptr(&heap_handle); + let heap_len = ctx.block().uitofp(I32, &heap_len_i32, DOUBLE); + let heap_end = ctx.block().label.clone(); + ctx.block().br(&done_label); + + ctx.current_block = sso_idx; + let sso_shifted = ctx.block().lshr(I64, &bits, SHORT_STRING_LEN_SHIFT); + let sso_len_byte = ctx.block().and(I64, &sso_shifted, "255"); + let sso_len = ctx.block().uitofp(I64, &sso_len_byte, DOUBLE); + let sso_end = ctx.block().label.clone(); + ctx.block().br(&done_label); + + ctx.current_block = done_idx; + let length = ctx + .block() + .phi(DOUBLE, &[(&heap_len, &heap_end), (&sso_len, &sso_end)]); + Ok(Some(length)) +} + +/// Emit ` ? A : B` as an f64 select over the two constants. +pub(crate) fn lower_cloned_bool_select( + ctx: &mut FnCtx<'_>, + read: &Expr, + on_true: i64, + on_false: i64, +) -> Result> { + let Some(exit) = side_exit_label(ctx, read) else { + return Ok(None); + }; + let Some(value) = emit_tracked_slot_word(ctx, read) else { + return Ok(None); + }; + + let bits = ctx.block().bitcast_double_to_i64(&value); + let is_true = ctx.block().icmp_eq(I64, &bits, TAG_TRUE_I64); + let is_false = ctx.block().icmp_eq(I64, &bits, TAG_FALSE_I64); + let is_bool = ctx.block().or(I1, &is_true, &is_false); + + let ok_idx = ctx.new_block("element_shape.bool"); + let ok_label = ctx.block_label(ok_idx); + // Everything that is not one of the two boolean singletons — `0`, `""`, + // `null`, an object — has a JS truthiness the clone is not allowed to + // guess, so it takes the side exit. + ctx.block().cond_br(&is_bool, &ok_label, &exit); + + ctx.current_block = ok_idx; + let then_literal = format!("{:.1}", on_true as f64); + let else_literal = format!("{:.1}", on_false as f64); + Ok(Some(ctx.block().select( + I1, + &is_true, + DOUBLE, + &then_literal, + &else_literal, + ))) +} + +/// The arithmetic-operand entry point (`expr::binary::lower_arithmetic_operand`). +/// +/// Returns the lowered f64, already representation-proven, so the caller must +/// NOT append a `js_number_coerce` — that call would delete the clone. +pub(crate) fn try_lower_cloned_read_operand( + ctx: &mut FnCtx<'_>, + expr: &Expr, +) -> Result> { + if let Some(read) = cloned_string_length_read(ctx, expr) { + let read = read.clone(); + return lower_cloned_string_length(ctx, &read); + } + if let Some((read, on_true, on_false)) = cloned_bool_select_read(ctx, expr) { + let read = read.clone(); + return lower_cloned_bool_select(ctx, &read, on_true, on_false); + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Anti-drift gate, the same one `element_shape_guard`'s constants carry: + /// every literal above reproduces a `perry-runtime::value::tags` constant, + /// and `perry-codegen` cannot share the definition. + #[test] + fn boolean_singleton_literals_match_the_runtime() { + assert_eq!(TAG_TRUE_I64, (0x7FFC_0000_0000_0004u64 as i64).to_string()); + assert_eq!(TAG_FALSE_I64, (0x7FFC_0000_0000_0003u64 as i64).to_string()); + // Neither may collide with any other singleton in the 0x7FFC namespace + // — a clone that read `undefined`/`null`/a hole as `false` would be a + // miscompile, not a slow path. + for other in [ + 0x7FFC_0000_0000_0001u64, // TAG_UNDEFINED + 0x7FFC_0000_0000_0002, // TAG_NULL + 0x7FFC_0000_0000_0010, // TAG_HOLE + 0x7FFC_0000_0000_0011, // TAG_TDZ + ] { + assert_ne!(TAG_TRUE_I64, (other as i64).to_string()); + assert_ne!(TAG_FALSE_I64, (other as i64).to_string()); + } + } + + #[test] + fn string_tag_literals_match_the_runtime() { + assert_eq!(STRING_TAG_TOP16, (0x7FFFu64).to_string()); + assert_eq!(SHORT_STRING_TAG_TOP16, (0x7FF9u64).to_string()); + assert_eq!(SHORT_STRING_LEN_SHIFT, "40"); + } + + #[test] + fn only_integral_ternary_arms_are_admitted() { + assert_eq!(integer_arm(&Expr::Integer(1)), Some(1)); + assert_eq!(integer_arm(&Expr::Number(0.0)), Some(0)); + assert_eq!(integer_arm(&Expr::Number(1.5)), None); + assert_eq!(integer_arm(&Expr::Number(f64::NAN)), None); + assert_eq!(integer_arm(&Expr::Number(f64::INFINITY)), None); + assert_eq!(integer_arm(&Expr::LocalGet(3)), None); + } +} diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index a0df121336..910ab2a688 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -156,6 +156,8 @@ mod array_push_guard_tests; mod barrier_stem_census_tests; #[cfg(test)] mod class_field_barrier_tests; +#[cfg(test)] +mod class_field_get_shape_tests; mod dispatch; #[cfg(test)] mod index_set_barrier_tests; @@ -2134,6 +2136,66 @@ pub(crate) enum ElementShapeIndex { /// Entry-block i32 alloca the clone writes the derived index to. slot: String, }, + /// #10185: a LOOP-CARRIED index — `let c = …;` outside the loop, + /// `c = (a*c + b) % m;` as the body's first statement, then `arr[c]` (or + /// `const index = c; arr[index]`). The benchmark's `random` mode, and the + /// shape every pseudo-random walk over a record array is written in. + /// + /// Unlike [`Self::DerivedMod`]'s `d`, `c` is a real mutable binding that is + /// LIVE AFTER THE LOOP, so the clone owes a write-back. It pays it once per + /// iteration, at the END of the iteration (`Carried::commit_slot`), which is + /// also what makes the residual side exit correct: every exit from the + /// middle of an iteration leaves the real slot holding the value it had at + /// that iteration's ENTRY, so the slow clone re-runs the update exactly + /// once. A write-back at the update site would double-apply the recurrence. + Carried(Box), +} + +/// #10185: everything [`ElementShapeIndex::Carried`] needs, boxed to keep the +/// enum small. +#[derive(Debug, Clone)] +pub(crate) struct CarriedIndex { + /// The `let` binding the recurrence advances. + pub local_id: u32, + /// `const index = c` — an optional alias the body may spell the subscript + /// through. Also virtual: its `Let` emits nothing. + pub alias_id: Option, + /// `c' = (a*c + b) % m`, folded to one affine pair by the matcher. Both + /// non-negative, and `|a| * i32::MAX + |b|` proven below 2^53 so the i64 + /// evaluation agrees with the f64 one JavaScript performs. + pub coeff_a: i64, + pub coeff_b: i64, + /// i32 SSA value of the modulus, materialized in the preheader. + pub modulus_i32: String, + /// Entry-block i32 alloca holding the live carried value inside the clone. + pub slot: String, + /// The binding's REAL (NaN-boxed double) slot, written once per iteration + /// after every side exit for that iteration has been passed. + pub commit_slot: String, +} + +/// #10185: the per-iteration element prefetch. +/// +/// `rows[index].id + rows[index].name.length + (rows[index].active ? 1 : 0)` +/// reads ONE element three times. Without this each read repeats the element +/// load AND the residual header/ShapeId check, which is three branches and +/// three redundant loads per iteration. The prologue attached to the body's +/// leading virtual binding does both once and parks the masked element handle +/// in an entry alloca; every read is then a bare offset load plus the tag test +/// its own consumer needs. +/// +/// It is also what makes a MULTI-READ body's side exit correct: every residual +/// exit now happens in the prologue, before any accumulator store has +/// committed, so resuming the iteration in the slow clone cannot double-apply +/// an earlier read. +#[derive(Debug, Clone)] +pub(crate) struct ElementPrefetch { + /// The virtual binding whose lowering emits the prologue. Exactly one + /// statement owns it, so a body with both a carried update and an alias + /// emits it once. + pub site_local_id: u32, + /// Entry-block i64 alloca holding the masked element handle. + pub handle_slot: String, } impl ElementShapeIndex { @@ -2152,7 +2214,22 @@ impl ElementShapeIndex { /// then declined would hand `is_numeric_expr` a raw-double promise the /// generic path does not keep. pub(crate) fn needs_counter_i32_slot(&self) -> bool { - !matches!(self, ElementShapeIndex::Constant(_)) + !matches!( + self, + ElementShapeIndex::Constant(_) | ElementShapeIndex::Carried(_) + ) + } + + /// #10185: the locals whose `Let` / `LocalSet` the fast clone lowers + /// VIRTUALLY for this index form. Nothing may read one of them bare inside + /// the clone, and their shadow slots must not be cleared there. + pub(crate) fn virtual_locals(&self) -> impl Iterator + '_ { + let (a, b) = match self { + ElementShapeIndex::DerivedMod { local_id, .. } => (Some(*local_id), None), + ElementShapeIndex::Carried(carried) => (Some(carried.local_id), carried.alias_id), + _ => (None, None), + }; + a.into_iter().chain(b) } } @@ -2200,6 +2277,10 @@ pub(crate) struct ElementShapeLoopFact { pub index_local_id: u32, /// #10123: how the clone computes the element index. pub index: ElementShapeIndex, + /// #10185: the shared per-iteration element deref, when the body has a + /// leading virtual binding to hang it off. `None` keeps #10123's per-read + /// deref + residual check. + pub elem_prefetch: Option, /// #10123: true when the preheader proved an exact ordinary ShapeId rather /// than a class id. The per-element residual check then drops the /// typed-layout conjunct (a parsed record's layout is @@ -2229,6 +2310,15 @@ pub(crate) struct ElementShapeLoopFact { /// raw-f64 candidates; shape-keyed ones are the preheader's live query /// results (#10123). pub fields: std::collections::BTreeMap, + /// #10185: true when the fast clone lowers a body the matcher SYNTHESIZED + /// (the K-statement accumulator fold, or the carried index's trailing + /// write-back) rather than the source body. The function-wide + /// shadow-slot-clear map is keyed by statement INDEX, so a rewritten body + /// would attribute a clear to the wrong statement; the clone is call-free, + /// so no collection can observe a slot inside it and the clears are simply + /// suppressed there (`expr::emit_shadow_slot_clear`). The slow clone lowers + /// the original body, with its original indices and its original clears. + pub synthesized_body: bool, /// #7771: the body's `const r = arr[counter]` binding, when the matcher /// admitted the element-binding form. Inside the fast clone the `Let` /// itself emits nothing (`stmt/let_stmt.rs`) and every `r.field` read @@ -2304,6 +2394,12 @@ pub(crate) fn element_shape_loop_fact_for_property_get<'f>( (ElementShapeIndex::DerivedMod { local_id, .. }, Expr::LocalGet(id)) => { *id == *local_id } + // #10185: `arr[c]` and `const index = c; arr[index]` are + // the same subscript — the alias is virtual, so both + // spellings read the carried slot the preheader bounded. + (ElementShapeIndex::Carried(carried), Expr::LocalGet(id)) => { + *id == carried.local_id || carried.alias_id == Some(*id) + } _ => false, }; if !spelled { @@ -2923,6 +3019,7 @@ mod typed_array_rmw; pub(crate) use instance_misc1::builtin_parent_reserved_class_id; pub(crate) mod class_field_inline_guard; pub(crate) mod element_shape_guard; +pub(crate) mod element_shape_reads; mod js_runtime; mod literals_vars; mod logical_collections; diff --git a/crates/perry-codegen/src/expr/property_get.rs b/crates/perry-codegen/src/expr/property_get.rs index b3b5b32c1c..51fe3d1214 100644 --- a/crates/perry-codegen/src/expr/property_get.rs +++ b/crates/perry-codegen/src/expr/property_get.rs @@ -1660,29 +1660,24 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { ); return Ok(val); } - // #5093: build the guard operands once, up front, so both - // the inline shape pre-check and the guard-call fallback - // can reference them. - let (obj_bits, obj_handle, key_raw) = { + // #5093: build the guard operands once, up front, so + // both the inline shape pre-check and the fast slot + // load can reference them. + let (obj_bits, obj_handle) = { let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(&recv_box); let obj_handle = blk.and(I64, &obj_bits, POINTER_MASK_I64); - let key_box = blk.load(DOUBLE, &key_handle_global); - let key_bits = blk.bitcast_double_to_i64(&key_box); - let key_raw = blk.and(I64, &key_bits, POINTER_MASK_I64); - (obj_bits, obj_handle, key_raw) + (obj_bits, obj_handle) }; let fast_idx = ctx.new_block("class_field_get.fast"); - let fallback_idx = ctx.new_block("class_field_get.fallback"); let merge_idx = ctx.new_block("class_field_get.merge"); let fast_label = ctx.block_label(fast_idx); - let fallback_label = ctx.block_label(fallback_idx); let merge_label = ctx.block_label(merge_idx); // #5093: inline shape pre-check. On a monomorphic hit it // branches straight to the fast slot load, skipping the // cross-crate guard call; on a miss it leaves the current - // block at the guard-call path below (unchanged). + // block at the single-exit IC call below. let subclass_arms = crate::expr::class_field_inline_guard::class_field_subclass_arms( ctx, @@ -1703,9 +1698,48 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { &fast_label, &subclass_arms, ); - let guard_ok = ctx.block().call( - I32, - "js_typed_feedback_class_field_get_guard", + // ONE EXIT. Everything the pre-check could not prove — + // the guard call, the guard-PASS slot load, the nullish + // TypeError, the fallback record and the by-name lookup — + // is the body of `js_class_field_get_ic`, the same helper + // the #5391 path-2 full outline a few lines above already + // calls. Emitting the four arms inline cost ~74 IR + // instructions and 4 runtime call sites per monomorphic + // `this.field` read; @babel/parser has 13,636 of them + // (27 % of its IR), and each call site is a statepoint, + // so `.perry_gcmap` scaled with them too. + // + // The pre-check and the fast slot load below are + // unchanged, so a pre-check HIT costs exactly what it + // cost before (no call at all) and a pre-check MISS costs + // exactly one call — what the guard call alone already + // cost. Arm for arm the helper reproduces the diamond: + // guard PASS -> reads the same slot at the same + // header-relative offset; + // nullish -> throws the same + // `js_throw_type_error_property_access` + // (#7153 added that check to the helper + // for exactly this equivalence); + // guard FAIL -> records the fallback, then + // `js_object_get_field_by_name_f64`. + // A plain number is self-boxing under NaN-boxing, so a + // `requires_raw_f64` site reads the helper's return the + // same way the old phi read the by-name fallback's return + // — neither arm converted, and neither does this one. + // + // The key handle is loaded HERE, in the cold miss block, + // instead of in the entry block: the call is its only + // consumer, so sinking it takes a load, a bitcast and a + // mask off the inline hit path. + let key_raw = { + let blk = ctx.block(); + let key_box = blk.load(DOUBLE, &key_handle_global); + let key_bits = blk.bitcast_double_to_i64(&key_box); + blk.and(I64, &key_bits, POINTER_MASK_I64) + }; + let val_ic = ctx.block().call( + DOUBLE, + "js_class_field_get_ic", &[ (I64, &site_id), (DOUBLE, &recv_box), @@ -1716,9 +1750,57 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { (I32, requires_raw_f64_str), ], ); - let guard_pass = ctx.block().icmp_ne(I32, &guard_ok, "0"); - ctx.block() - .cond_br(&guard_pass, &fast_label, &fallback_label); + let ic_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + if requires_raw_f64 { + // The miss arm's value now arrives from the outlined + // IC instead of the inline by-name call, so the + // dynamic-fallback record follows it. Same contract + // as before — a JS value, not a proven raw double, + // with the raw-f64 layout rejected and invalidated by + // a runtime API (`native_value/verify/raw_f64.rs` + // enforces all three on this consumer). + let fallback = LoweredValue { + semantic: SemanticKind::JsValue, + rep: NativeRep::JsValue, + llvm_ty: DOUBLE, + value: val_ic.clone(), + }; + ctx.record_lowered_value_with_access_mode_and_facts( + "ClassFieldGet", + None, + "js_class_field_get_ic", + &fallback, + Some(BoundsState::Unknown), + None, + Some(BufferAccessMode::DynamicFallback), + Some(MaterializationReason::RuntimeApi), + None, + None, + Vec::new(), + vec![ + raw_f64_layout_fact( + None, + "rejected", + "class_field_get_guard", + Some(MaterializationReason::RuntimeApi), + ), + raw_f64_layout_fact( + None, + "invalidated", + "runtime_api", + Some(MaterializationReason::RuntimeApi), + ), + ], + false, + false, + vec![ + format!("class={}", class_name), + format!("field={}", property), + format!("field_index={}", field_idx_str), + ], + ); + } ctx.current_block = fast_idx; // arm64_32 watchOS: the object fields region begins at @@ -1782,109 +1864,10 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { ); } - ctx.current_block = fallback_idx; - // #7153: the guard rejects a nullish receiver along with - // every other shape miss, but a nullish field read must - // throw TypeError per spec — the by-name lookup below - // answers `undefined` and the program keeps running on a - // silent wrong value. Mirror the generic path's check - // (generic_dispatch.rs); the cost sits on the cold - // fallback arm only. - let (is_null, is_nullish) = { - let blk = ctx.block(); - let is_undef = - blk.icmp_eq(I64, &obj_bits, crate::nanbox::TAG_UNDEFINED_I64); - let is_null = blk.icmp_eq(I64, &obj_bits, crate::nanbox::TAG_NULL_I64); - let is_nullish = blk.or(I1, &is_undef, &is_null); - (is_null, is_nullish) - }; - let throw_idx = ctx.new_block("class_field_get.throw_nullish"); - let lookup_idx = ctx.new_block("class_field_get.fallback_lookup"); - let throw_label = ctx.block_label(throw_idx); - let lookup_label = ctx.block_label(lookup_idx); - ctx.block() - .cond_br(&is_nullish, &throw_label, &lookup_label); - - ctx.current_block = throw_idx; - let prop_entry = ctx.strings.entry(key_idx); - let prop_bytes_global = format!("@{}", prop_entry.bytes_global); - let prop_len_str = prop_entry.byte_len.to_string(); - let is_null_i32 = ctx.block().zext(I1, &is_null, I32); - ctx.block().call_void( - "js_throw_type_error_property_access", - &[ - (I32, &is_null_i32), - (PTR, &prop_bytes_global), - (I64, &prop_len_str), - ], - ); - ctx.block().unreachable(); - - ctx.current_block = lookup_idx; - let blk = ctx.block(); - crate::expr::emit_typed_feedback_record_call( - blk, - "js_typed_feedback_record_fallback_call", - &[(I64, &site_id)], - ); - let val_fallback_js = blk.call( - DOUBLE, - "js_object_get_field_by_name_f64", - &[(I64, &obj_bits), (I64, &key_raw)], - ); - let val_fallback = val_fallback_js.clone(); - let fallback_end_label = blk.label.clone(); - blk.br(&merge_label); - if requires_raw_f64 { - let fallback = LoweredValue { - semantic: SemanticKind::JsValue, - rep: NativeRep::JsValue, - llvm_ty: DOUBLE, - value: val_fallback_js.clone(), - }; - ctx.record_lowered_value_with_access_mode_and_facts( - "ClassFieldGet", - None, - "js_object_get_field_by_name_f64", - &fallback, - Some(BoundsState::Unknown), - None, - Some(BufferAccessMode::DynamicFallback), - Some(MaterializationReason::RuntimeApi), - None, - None, - Vec::new(), - vec![ - raw_f64_layout_fact( - None, - "rejected", - "class_field_get_guard", - Some(MaterializationReason::RuntimeApi), - ), - raw_f64_layout_fact( - None, - "invalidated", - "runtime_api", - Some(MaterializationReason::RuntimeApi), - ), - ], - false, - false, - vec![ - format!("class={}", class_name), - format!("field={}", property), - format!("field_index={}", field_idx_str), - ], - ); - } - ctx.current_block = merge_idx; return Ok(ctx.block().phi( DOUBLE, - &[ - (&val_fast, &fast_end_label), - (&val_fallback, &fallback_end_label), - ], + &[(&val_fast, &fast_end_label), (&val_ic, &ic_end_label)], )); } } diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 28cb608ffe..889e059eaa 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -157,6 +157,32 @@ pub(crate) fn lower_generic_property_get( return Ok(val); } + // # Inline hit, two exits (T1) + // + // What stays inline below is exactly the hit: the receiver-tag test, the + // small-handle test, the packed header kind/descriptor word, the compact + // MRU compare, the overflow-bit test, the raw slot load with its hole + // check, and the bounded polymorphic ways. EVERY other arm this tower used + // to expand — the SSO receiver, the INT32 class ref, the nullish throw, the + // non-object receiver, the overflow load, the deleted-slot miss, the two + // Array-subclass named-prefix ladders, and the miss+prime — is now a branch + // to one of TWO calls that reproduce them in the same order: + // `js_object_get_field_ic_nonptr` for a receiver that is not a heap + // pointer, `js_object_get_field_ic_slow` for one that is. The split is not + // cosmetic — see `pget.recv_other` below for the +4 instructions per HIT + // that a single shared exit cost. + // + // The arms were not cheap to keep: ~37 basic blocks, ~177 pre-RS4GC IR + // instructions and 6-7 call sites per site, each call a statepoint whose + // live GC values are written into `.perry_gcmap`. On @babel/parser the + // tower was 29% of all emitted IR across 6,487 sites. It is also not a + // trade against the fast path: the hit sequence below is instruction for + // instruction what it was, with ONE deliberate difference — the + // overflow-bit test is spelled `== 0` with its successors swapped, so the + // guard-passing edge is the true edge like every other link in the chain + // (see `pic.hit` below) — and the ways still resolve `PIC_WAYS + 1` shapes + // without a call. + // // Issue #70/#73/#128: guard against non-pointer receivers // before the PIC deref. Tag-based check on the unmasked // NaN-box: real heap references have high-16-bits POINTER_TAG @@ -164,8 +190,9 @@ pub(crate) fn lower_generic_property_get( // to 0x7FFD; everything else (undefined/null/bool=0x7FFC, // int32=0x7FFE, bigint=0x7FFA, plain f64 like 0.0 globalThis // or 3.14, corrupt bit-patterns like 0x00FF_0000_0000 read as - // a BufferHeader) falls through to the invalid branch and - // returns undefined safely. + // a BufferHeader) falls through to the slow entry, whose tag + // ladder throws for a nullish receiver (#462) and routes every + // other tag to the by-name helper. // // Previously used a Darwin mimalloc heap-window check // (`> 2 TB && < 128 TB`). On aarch64-linux-android (issue @@ -176,41 +203,9 @@ pub(crate) fn lower_generic_property_get( // Tag check is platform-independent: same two LLVM ops // (`lshr` + `and`) + one `icmp`, branch-predicted taken. let obj_tag = ctx.block().lshr(I64, &obj_bits, "48"); - // SSO receiver fast path (Step 1.5 of SSO migration). - // SHORT_STRING_TAG = 0x7FF9 can't pass the POINTER/STRING - // check (its masked tag is 0x7FF9, not 0x7FFD) and we - // can't widen the mask because the PIC fast path's - // `*(obj_handle + 16)` would read arbitrary memory from - // the SSO data bits. Instead: check SSO explicitly first, - // route to a dedicated block that calls the SSO-aware - // `js_object_get_field_by_name_f64` runtime entry (which - // handles `.length` directly from the NaN-box length - // byte and returns `undefined` for other keys). - // v0.5.747: INT32-tagged class refs (top16 == 0x7FFE) used - // as PropertyGet receivers. Pre-fix these fell through to - // the invalid-recv path (returning undefined) because the - // 0xFFFD-masked tag check (0x7FFE & 0xFFFD = 0x7FFC, not - // 0x7FFD) treated them as non-pointer values. Drizzle's - // `is(value, type)` chain depends on `Cls.kind` reads through - // an Any-typed local. Refs #420 / #618 followup. - // - // Note: this also catches plain int32 numeric values (e.g. - // `(42).property`). The runtime helper's INT32-tag arm at - // js_object_get_field_by_name returns undefined for any - // class_id not registered in CLASS_DYNAMIC_PROPS, matching - // the previous behavior — pure ints have no static fields. let obj_tag_masked = ctx.block().and(I64, &obj_tag, "65533"); // 0xFFFD let is_valid = ctx.block().icmp_eq(I64, &obj_tag_masked, "32765"); // 0x7FFD - let sso_idx = ctx.new_block("pget.recv_sso"); - let pic_idx = ctx.new_block("pget.recv_ok"); - let invalid_idx = ctx.new_block("pget.recv_bad"); - let class_ref_idx = ctx.new_block("pget.recv_class_ref"); - let final_merge_idx = ctx.new_block("pget.recv_merge"); - let sso_label = ctx.block_label(sso_idx); - let pic_label = ctx.block_label(pic_idx); - let invalid_label = ctx.block_label(invalid_idx); - let class_ref_label = ctx.block_label(class_ref_idx); - let final_merge_label = ctx.block_label(final_merge_idx); + // `.length` on a receiver whose static type is not a proven string. // // The three-arm string-length dispatch in `property_get.rs` (SSO length @@ -221,11 +216,8 @@ pub(crate) fn lower_generic_property_get( // type (`rec.tag.length` where `rec` is an object-literal type, a JSON // `any`, an array element) therefore lands in this generic tower instead, // where a heap string can never be served: the PIC requires a - // GC_TYPE_OBJECT receiver by construction (#72), so EVERY such read misses - // to `js_object_get_field_ic_miss` and walks a ladder built for objects — - // closure-magic deref, buffer and typed-array registry probes, then - // `js_object_get_field_by_name`'s own dispatch, which decodes the key with - // `str::from_utf8` again before reaching the string arm. On `pipeline.ts` + // GC_TYPE_OBJECT receiver by construction (#72), so EVERY such read would + // otherwise call out and walk a ladder built for objects. On `pipeline.ts` // that one read was ~9% of total run time. // // Both string tags are disjoint from POINTER_TAG, so serving them here is @@ -233,11 +225,6 @@ pub(crate) fn lower_generic_property_get( // non-configurable, cannot be shadowed by an own property, and is exactly // what the runtime ladder computes. Everything else keeps the tower. let inline_string_length = property == "length"; - let strlen_heap_idx = if inline_string_length { - Some(ctx.new_block("pget.strlen_heap")) - } else { - None - }; // A dynamically typed `receiver.size` can still be served without the // object PIC when the live receiver is a native Map or Set. Both payloads // start with the same `u32 size` field, and their distinct GcHeader kinds @@ -246,52 +233,105 @@ pub(crate) fn lower_generic_property_get( // as `this.ctx.hooks.size` commonly lose their static Set type, while an // erased annotation alone must never authorize a native-layout load. let inline_collection_size = property == "size"; - let collection_size_idx = if inline_collection_size { - Some(ctx.new_block("pget.collection_size")) - } else { - None - }; - // #7883: the POINTER/STRING test goes FIRST, and the two rare tags are - // discriminated in a cold block off its false edge. The three tag classes - // are pairwise disjoint — `is_valid` is `(tag & 0xFFFD) == 0x7FFD`, true - // only for 0x7FFD/0x7FFF, while SSO is 0x7FF9 and an INT32 class ref is - // 0x7FFE — so testing them in any order gives the same routing. The old - // order (SSO, then class-ref, then pointer) put two 16-bit constant - // materialisations, two compares and two branches in front of every real - // object receiver: 13 instructions before the PIC on the path that is - // taken essentially always. Now it is `lshr` + `and` + `cmp` + branch. + + // A compact per-site word holds the exact ShapeId and slot for the last + // cacheable receiver. The lazily allocated full cache retains bounded + // polymorphic ways and the Array-subclass named-prefix proof. Both are + // materialised before the first branch now: the single slow exit takes + // them as arguments, and every failing guard reaches it. + let cache_name = overridden_cache_name(ctx, object, property) + .unwrap_or_else(|| allocate_property_cache(ctx)); + let cache_slot_ref = format!("@{cache_name}"); + // A compact atomic MRU removes the cache-pointer dependency on a hit. + // The full cache stays lazy and serves prefix/overflow/polymorphic misses. + let packed_site = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let packed_name = format!( + "{}_packed_get", + crate::expr::inline_cache_global_name(ctx, packed_site) + ); + ctx.typed_parse_rodata + .push(format!("@{packed_name} = private global i64 0, align 8")); + let packed_ref = format!("@{packed_name}"); + + let pic_idx = ctx.new_block("pget.recv_ok"); + // The object exit. Named `pic.miss.call` because that is what it still is: + // the block that calls out when the inline cache cannot serve the read. It + // now also lands every receiver-validation failure ON THE POINTER PATH, + // which is what collapses five of the six old call sites into it. + let call_idx = ctx.new_block("pic.miss.call"); + // The non-pointer exit. Its own block and its own callee, deliberately: + // when the receiver-tag test and the small-handle test failed to the SAME + // block, SimplifyCFG folded the two guards into one flat predicate + // (`cmp; sete; cmp; setae; test; je` for `cmp; jne; cmp; ja`) and every + // property-read HIT paid +4.00 instructions — measured on a 10M-read + // monomorphic loop, 1,231,521,261 -> 1,271,522,669 retired. That is + // #7883's flat-predicate cost arriving from the optimiser instead of from + // codegen. Distinct callees keep the chain branchy, and as a bonus the + // unmasked `obj_bits` dies here instead of staying live across the hit + // path for a call that might need it. let other_idx = ctx.new_block("pget.recv_other"); + let merge_idx = ctx.new_block("pget.recv_merge"); + let pic_label = ctx.block_label(pic_idx); + let call_label = ctx.block_label(call_idx); let other_label = ctx.block_label(other_idx); - let check_class_ref_idx = ctx.new_block("pget.check_class_ref"); - let check_class_ref_label = ctx.block_label(check_class_ref_idx); + let merge_label = ctx.block_label(merge_idx); + + // Typed-feedback bookkeeping is COMPILE-TIME gated (off by default), so + // the recording landing block only exists when it has something to record. + // Without it every failing guard branches straight to `pic.miss.call`; + // with it, the guard-fail/fallback-call pair is recorded on exactly the + // edges that recorded it before (`pic.miss.cold` and `pic.miss`), and the + // edges that recorded nothing — the non-pointer tags, the overflow slot, + // a way that turned out to hold a hole — still record nothing. + let cold_idx = + crate::expr::typed_feedback_emission_enabled().then(|| ctx.new_block("pic.miss.cold")); + let cold_label = cold_idx + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| call_label.clone()); + ctx.block().cond_br(&is_valid, &pic_label, &other_label); + + // Off the pointer path only ONE arm is still worth inlining: an SSO + // string's `.length`, which is a byte of the NaN-box itself. Every other + // non-pointer receiver — an SSO string with any other key, an INT32 class + // ref, `null`/`undefined`, a plain double — is the non-pointer entry's tag + // ladder, in the same order it evaluated them here. ctx.current_block = other_idx; - let is_sso = ctx.block().icmp_eq(I64, &obj_tag, "32761"); // 0x7FF9 - ctx.block() - .cond_br(&is_sso, &sso_label, &check_class_ref_label); - ctx.current_block = check_class_ref_idx; - let is_int32_class = ctx.block().icmp_eq(I64, &obj_tag, "32766"); // 0x7FFE - ctx.block() - .cond_br(&is_int32_class, &class_ref_label, &invalid_label); + let sso_arm = inline_string_length.then(|| { + let sso_idx = ctx.new_block("pget.recv_sso"); + let sso_label = ctx.block_label(sso_idx); + let nonptr_idx = ctx.new_block("pget.recv_nonptr"); + let nonptr_label = ctx.block_label(nonptr_idx); + let is_sso = ctx.block().icmp_eq(I64, &obj_tag, "32761"); // 0x7FF9 + ctx.block().cond_br(&is_sso, &sso_label, &nonptr_label); - // Class-ref dispatch: route through the runtime helper which - // detects INT32 class-ref bits and consults CLASS_DYNAMIC_PROPS - // for the static field / dynamic IIFE-set property / synthetic - // `constructor` lookup. Pass full obj_bits (NOT obj_handle — - // the runtime needs the unmasked top16 to detect the tag). - ctx.current_block = class_ref_idx; - let key_handle = emit_key_handle(ctx, &key_handle_global); - let class_ref_result = ctx.block().call( + // `.length` of an SSO string is the length byte in bits 40..47 of the + // NaN-box itself — the same extract `js_object_get_field_by_name_f64` + // performs, minus the call and the key decode. + ctx.current_block = sso_idx; + let len_shifted = ctx.block().lshr(I64, &obj_bits, "40"); + let len_byte = ctx.block().and(I64, &len_shifted, "255"); + let sso_val = ctx.block().uitofp(I64, &len_byte, DOUBLE); + let sso_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + ctx.current_block = nonptr_idx; + (sso_val, sso_end_label) + }); + // The non-pointer exit: SSO / INT32 class ref / nullish throw / everything + // else, in that order, behind one call that needs no cache. + let nonptr_key_handle = emit_key_handle(ctx, &key_handle_global); + let val_nonptr = ctx.block().call( DOUBLE, - "js_typed_feedback_object_get_field_by_name_f64", + "js_object_get_field_ic_nonptr", &[ - (I64, &feedback_site_id), (I64, &obj_bits), - (I64, &key_handle), + (I64, &nonptr_key_handle), + (I64, &feedback_site_id), ], ); - let class_ref_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); + let nonptr_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); ctx.current_block = pic_idx; let observed_key = key_handle_observed.clone().unwrap_or_default(); @@ -309,7 +349,8 @@ pub(crate) fn lower_generic_property_get( // typed-feedback observation on purpose: the site keeps recording every // receiver it sees, so a mixed object/string site cannot be mis-profiled // as monomorphic-object by the arm that is no longer traced here. - if let Some(heap_idx) = strlen_heap_idx { + let strlen_heap_idx = inline_string_length.then(|| { + let heap_idx = ctx.new_block("pget.strlen_heap"); let strlen_heap_label = ctx.block_label(heap_idx); let not_string_idx = ctx.new_block("pget.recv_obj"); let not_string_label = ctx.block_label(not_string_idx); @@ -319,24 +360,8 @@ pub(crate) fn lower_generic_property_get( ctx.block() .cond_br(&is_heap_string, &strlen_heap_label, ¬_string_label); ctx.current_block = not_string_idx; - } - - // A compact per-site word holds the exact ShapeId and slot for the last - // cacheable receiver. The lazily allocated full cache retains bounded - // polymorphic ways and the Array-subclass named-prefix proof. - let cache_name = overridden_cache_name(ctx, object, property) - .unwrap_or_else(|| allocate_property_cache(ctx)); - // A compact atomic MRU removes the cache-pointer dependency on a hit. - // The full cache stays lazy and serves prefix/overflow/polymorphic misses. - let packed_site = ctx.ic_site_counter; - ctx.ic_site_counter += 1; - let packed_name = format!( - "{}_packed_get", - crate::expr::inline_cache_global_name(ctx, packed_site) - ); - ctx.typed_parse_rodata - .push(format!("@{packed_name} = private global i64 0, align 8")); - let packed_ref = format!("@{packed_name}"); + heap_idx + }); // Issue #72: validate the receiver is actually a GC_TYPE_OBJECT // before reading its ShapeId. The receiver @@ -346,20 +371,19 @@ pub(crate) fn lower_generic_property_get( // analysis can't prove `obj.rowsRaw` is an Array — the outer // PropertyGet falls into this generic dispatch) hands the array's // pointer to this PIC. Reading an ObjectHeader ShapeId from that payload - // would be invalid. The slow `js_object_get_field_by_name` - // already routes by `gc_type` (handles Array.length, String.length, - // Set.size, Buffer.length, Error.message, etc.), so funneling - // non-OBJECT receivers through the miss handler fixes correctness - // without giving up the PIC for real objects. + // would be invalid. The slow entry already routes by `gc_type` (it handles + // Array.length, String.length, Set.size, Buffer.length, Error.message, + // etc. through the same miss handler), so funneling non-OBJECT receivers + // to it fixes correctness without giving up the PIC for real objects. // // Issue #340/#341: small-handle guard. Receivers from // native modules (axios, fastify, ioredis, better-sqlite3, // ...) are NaN-boxed POINTER values whose lower-48 is a // small registry id (1, 2, 3, ...). The PIC fast path // below deref's `obj_handle - 8` for the GcHeader byte - // and `obj_handle + 8` for the ShapeId slot — both + // and `obj_handle + 4` for the ShapeId — both // SIGSEGV when `obj_handle` is a small int. Funnel - // small-handle receivers through the slow path so they + // small-handle receivers through the slow entry so they // reach the runtime's `HANDLE_PROPERTY_DISPATCH` table // (axios `r.status` / `r.data`, fastify `req.query` / // `req.params`, etc.). @@ -368,60 +392,43 @@ pub(crate) fn lower_generic_property_get( // detection (raw_ptr < 0x100000). let is_real_ptr = ctx.block().icmp_ugt(I64, &obj_handle, "1048575"); // 0x100000 - // #7883: the hit/miss/merge blocks are minted here so the guard chain - // below can BRANCH OUT to the miss on the first failing predicate - // instead of AND-ing eight of them into one flat `hit`. LLVM if-converts - // a flat predicate, so every receiver paid every load and every compare - // even after the very first one had - // already decided the answer. Each group now ends in its own `cond_br`; - // the miss block reconstructs what the polymorphic-way compares need - // through phis (`false`/`0` on the early-exit edges, which is exactly - // what the flat predicate computed there). - let hit_idx = ctx.new_block("pic.hit"); - let hit_live_idx = ctx.new_block("pic.hit.live"); - let prefix_guard_idx = ctx.new_block("pic.prefix.guard"); - let prefix_meta_idx = ctx.new_block("pic.prefix.meta"); - let prefix_token_idx = ctx.new_block("pic.prefix.token"); - let prefix_hit_idx = ctx.new_block("pic.prefix.hit"); - let desc_classify_idx = ctx.new_block("pic.desc.classify"); - let desc_prefix_guard_idx = ctx.new_block("pic.desc.prefix.guard"); - let desc_prefix_meta_idx = ctx.new_block("pic.desc.prefix.meta"); - let desc_prefix_token_idx = ctx.new_block("pic.desc.prefix.token"); - let desc_prefix_hit_idx = ctx.new_block("pic.desc.prefix.hit"); - let miss_idx = ctx.new_block("pic.miss"); - // #7907: the two receiver-validation failures get their own landing block - // so `pic.miss` is dominated by `pic.token`. See the comment on - // `pic.miss.cold` below for why that is the whole point of this split. - let cold_idx = ctx.new_block("pic.miss.cold"); - let call_idx = ctx.new_block("pic.miss.call"); - let merge_idx = ctx.new_block("pic.merge"); - let hit_label = ctx.block_label(hit_idx); - let hit_live_label = ctx.block_label(hit_live_idx); - let prefix_guard_label = ctx.block_label(prefix_guard_idx); - let prefix_meta_label = ctx.block_label(prefix_meta_idx); - let prefix_token_label = ctx.block_label(prefix_token_idx); - let prefix_hit_label = ctx.block_label(prefix_hit_idx); - let desc_classify_label = ctx.block_label(desc_classify_idx); - let desc_prefix_guard_label = ctx.block_label(desc_prefix_guard_idx); - let desc_prefix_meta_label = ctx.block_label(desc_prefix_meta_idx); - let desc_prefix_token_label = ctx.block_label(desc_prefix_token_idx); - let desc_prefix_hit_label = ctx.block_label(desc_prefix_hit_idx); - let miss_label = ctx.block_label(miss_idx); - let cold_label = ctx.block_label(cold_idx); - let call_label = ctx.block_label(call_idx); - let merge_label = ctx.block_label(merge_idx); + // #7883: the guard chain BRANCHES OUT to the slow exit on the first + // failing predicate instead of AND-ing eight of them into one flat `hit`. + // LLVM if-converts a flat predicate, so every receiver paid every load and + // every compare even after the very first one had already decided the + // answer. let hdr_idx = ctx.new_block("pic.recv_hdr"); let hdr_label = ctx.block_label(hdr_idx); let tok_idx = ctx.new_block("pic.token"); let tok_label = ctx.block_label(tok_idx); + let hit_idx = ctx.new_block("pic.hit"); + // The inline hit's LIVE edge goes straight to the merge unless typed + // feedback has a guard-pass record to put on it. An empty `pic.hit.live` + // is congruent with `pic.way.live`, so SimplifyCFG tail-merges the two and + // the hit path pays a `jmp` to the survivor instead of falling through. + let hit_live_idx = + crate::expr::typed_feedback_emission_enabled().then(|| ctx.new_block("pic.hit.live")); + // The hole edge gets its own landing block, as it did before T1. Note what + // that does and does not buy, measured: `pic.hit.inline` and `pic.way.load` + // end in the same three instructions (bitcast, TAG_HOLE compare, branch), + // and SimplifyCFG folds this block and `pic.way.live` away, so the two + // tails end up congruent and get merged anyway — the hit still reaches the + // shared tail by a `jmp`, and removing this block changed the 10M-read + // monomorphic loop by exactly 0 instructions. It is kept because it keeps + // the emitted hole edge structurally distinct from the way path's, which is + // the shape every reader of this tower since #9287 expects; the remaining + // jump needs branch weights (`!prof`) to fix, which the IR builder has no + // way to emit today. + let deleted_idx = ctx.new_block("pic.hit.deleted"); + let miss_idx = ctx.new_block("pic.miss"); + let hit_label = ctx.block_label(hit_idx); + let deleted_label = ctx.block_label(deleted_idx); + let miss_label = ctx.block_label(miss_idx); // Small-handle receivers (native-module registry ids) must never be // dereferenced. Pre-#7883 they were kept out of the loads by selecting a // sentinel address and AND-ing `is_real_ptr` into `hit`; the branch does // the same job without putting a `select` (and the sentinel's address // materialisation) in front of every real object read. - // A small-handle receiver can never resolve a way (`way_hit` requires a - // real object), so it leaves for `pic.miss.cold` and never enters the - // block the ways live in. ctx.block().cond_br(&is_real_ptr, &hdr_label, &cold_label); ctx.current_block = hdr_idx; @@ -440,33 +447,42 @@ pub(crate) fn lower_generic_property_get( "aarch64" | "arm64" | "arm64_32" | "x86_64" | "i686" | "i386" | "riscv64" | "wasm32" ) .then(|| ctx.block().load(I32, &gc_type_ptr)); - let gc_type = match &packed_header { - Some(word) => ctx.block().trunc(I32, word, I8), - None => ctx.block().load(I8, &gc_type_ptr), - }; + // The separate kind BYTE is needed only by the native-endian descriptor + // guard and by the Map/Set split. With the packed header word the kind is + // already inside it, so materialising the byte anyway would leave a dead + // `trunc` on every hit. + let gc_type = + (inline_collection_size || packed_header.is_none()).then(|| match &packed_header { + Some(word) => ctx.block().trunc(I32, word, I8), + None => ctx.block().load(I8, &gc_type_ptr), + }); // `MapHeader` and `SetHeader` both begin with `size: u32`. A native // collection is not an ObjectHeader and can never hit this PIC, so split // it off immediately after the already-required GC-kind load. The generic // miss handler recognizes the same two kinds before ordinary object // lookup; this only removes that repeated classification and call ladder. - if let Some(collection_idx) = collection_size_idx { + let collection_size_idx = inline_collection_size.then(|| { + let collection_idx = ctx.new_block("pget.collection_size"); let collection_label = ctx.block_label(collection_idx); let object_check_idx = ctx.new_block("pic.recv_object_check"); let object_check_label = ctx.block_label(object_check_idx); - let is_map = ctx.block().icmp_eq(I8, &gc_type, "8"); // GC_TYPE_MAP - let is_set = ctx.block().icmp_eq(I8, &gc_type, "12"); // GC_TYPE_SET + let kind = gc_type + .clone() + .expect("the GC-kind byte is materialised whenever `.size` is inlined"); + let is_map = ctx.block().icmp_eq(I8, &kind, "8"); // GC_TYPE_MAP + let is_set = ctx.block().icmp_eq(I8, &kind, "12"); // GC_TYPE_SET let is_collection = ctx.block().or(I1, &is_map, &is_set); ctx.block() .cond_br(&is_collection, &collection_label, &object_check_label); ctx.current_block = object_check_idx; - } - let is_object_kind = ctx.block().icmp_eq(I8, &gc_type, "2"); + collection_idx + }); // Closures and RegExp values have distinct GC kinds. Every // `GC_TYPE_OBJECT` payload is therefore an ObjectHeader and its ShapeId is // the remaining exact layout discriminator. - + // // #6080: a receiver that has ever had a property/accessor descriptor // installed (`Object.defineProperty`) needs descriptor-aware dispatch — // an accessor must fire on reads, a non-writable slot must reject stores. @@ -476,10 +492,18 @@ pub(crate) fn lower_generic_property_get( // hit path would return the raw slot and bypass the getter entirely. // OBJ_FLAG_HAS_DESCRIPTORS is bit 11 of reserved (bit 27 of the // little-endian header word). Ignore gc_flags and every other flag. + // A descriptor-bearing receiver leaves for the slow exit, which keeps the + // Array-subclass named-prefix proof that used to be a second inline ladder + // here: it is the one case where an unrelated `length` descriptor must not + // make every declared field permanently generic. let is_plain_kind = if let Some(word) = &packed_header { let kind_and_desc = ctx.block().and(I32, word, "134217983"); // 0x080000ff ctx.block().icmp_eq(I32, &kind_and_desc, "2") } else { + let kind = gc_type + .as_ref() + .expect("the GC-kind byte is materialised on native-endian targets"); + let is_object_kind = ctx.block().icmp_eq(I8, kind, "2"); let reserved_addr = ctx.block().sub(I64, &obj_handle, "6"); let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); let reserved = ctx.block().load(crate::types::I16, &reserved_ptr); @@ -487,33 +511,12 @@ pub(crate) fn lower_generic_property_get( let no_desc = ctx.block().icmp_eq(crate::types::I16, &has_desc, "0"); ctx.block().and(I1, &is_object_kind, &no_desc) }; - // Resolve the full cache only after the compact MRU declines a read. - // Loading it here keeps an unused global load on every successful hit. - // Each cold entry retains its own non-null proof before dereferencing. - let cache_slot_ref = format!("@{cache_name}"); let is_plain_object = ctx.block().and(I1, &is_plain_kind, &packed_present); // Validate kind, descriptor policy, and initialized MRU before reading - // ObjectHeader's ShapeId. Header failures bypass the shape ways; the - // descriptor-aware named-prefix path keeps its own full-cache guard. + // ObjectHeader's ShapeId. ctx.block() - .cond_br(&is_plain_object, &tok_label, &desc_classify_label); - - // A descriptor-bearing GC_TYPE_OBJECT normally goes cold. Array-subclass - // `length` is the important exception: runtime can prove that descriptor - // is unrelated to all class-declared named fields and arm word 2. Keep - // this classification off the ordinary descriptor-free hit path. - ctx.current_block = desc_classify_idx; - // #9708: the descriptor prefix path reads cache word 2, so it needs the - // full-cache non-null proof as the shape-miss path; without a cache it is - // simply a cold miss. - let desc_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); - let desc_object_with_cache = ctx.block().and(I1, &is_object_kind, &desc_cache.present); - ctx.block().cond_br( - &desc_object_with_cache, - &desc_prefix_guard_label, - &cold_label, - ); + .cond_br(&is_plain_object, &tok_label, &cold_label); ctx.current_block = tok_idx; // The receiver token is derived solely from its authoritative ShapeId. @@ -539,11 +542,14 @@ pub(crate) fn lower_generic_property_get( let token_miss_label = ctx.block_label(token_miss_idx); ctx.block() .cond_br(&token_eq, &hit_label, &token_miss_label); + + // Every way load still requires a resolved full cache. A site that has + // never primed has no cache, so there is nothing to compare against and + // the read goes straight out. ctx.current_block = token_miss_idx; - // Every subsequent prefix/way load still requires a resolved full cache. let token_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); ctx.block() - .cond_br(&token_cache.present, &prefix_guard_label, &cold_label); + .cond_br(&token_cache.present, &miss_label, &cold_label); // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact @@ -555,216 +561,86 @@ pub(crate) fn lower_generic_property_get( // #9287: the primed slot word may carry IC_SLOT_OVERFLOW_BIT (1 << 30) — // the field lives past the inline region, in the object's spill buffer, // and the inline `obj + header + slot*8` arithmetic below must not run on - // it. Such hits route through `js_object_get_field_ic_overflow_load`, - // which loads through `overflow_get` and falls back to the full miss - // handler on a tombstoned slot. Sites whose field is inline never see the - // bit, so this branch predicts perfectly for them. The polymorphic WAYS - // never hold an encoded slot (`pic_prime_get` refuses to cascade one), so - // only this MRU path needs the check. - let ovf_idx = ctx.new_block("pic.hit.overflow"); + // it. Such hits leave for the slow entry, which re-derives the same MRU + // pair and performs `js_object_get_field_ic_overflow_load`'s `overflow_get` + // (falling back to the full miss handler on a tombstoned slot, without the + // packed republication that helper also did not do). Sites whose field is + // inline never see the bit, so this branch predicts perfectly for them. + // The polymorphic WAYS never hold an encoded slot (`pic_prime_get` refuses + // to cascade one), so only this MRU path needs the check. + // + // Tested as `== 0` rather than `!= 0` so the guard-PASSING edge is the TRUE + // edge, exactly like every other link in this chain. That is not cosmetic: + // `generic_property_get_slot_load_is_reached_only_through_every_guard` + // walks the CFG backwards from the slot load and requires every edge on the + // way to be a true edge, which is what makes a swapped `cond_br` — running + // the raw load when a guard FAILS — turn it red. Before T1 the walk reached + // the load through `pic.hit.overflow` (whose key-handle load also matched + // "load double") and never evaluated this branch's polarity at all. let inline_hit_idx = ctx.new_block("pic.hit.inline"); - let ovf_label = ctx.block_label(ovf_idx); let inline_hit_label = ctx.block_label(inline_hit_idx); let ovf_bits = ctx.block().and(I64, &slot, "1073741824"); // 1 << 30 - let is_ovf = ctx.block().icmp_ne(I64, &ovf_bits, "0"); - ctx.block().cond_br(&is_ovf, &ovf_label, &inline_hit_label); - - ctx.current_block = ovf_idx; - let ovf_key_handle = emit_key_handle(ctx, &key_handle_global); - let ovf_slot_i32 = ctx.block().trunc(I64, &slot, I32); - let val_ovf = ctx.block().call( - DOUBLE, - "js_object_get_field_ic_overflow_load", - &[ - (I64, &obj_handle), - (I64, &ovf_key_handle), - (I32, &ovf_slot_i32), - (PTR, &cache_slot_ref), - ], - ); - let ovf_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); + let is_inline_slot = ctx.block().icmp_eq(I64, &ovf_bits, "0"); + ctx.block() + .cond_br(&is_inline_slot, &inline_hit_label, &call_label); ctx.current_block = inline_hit_idx; - let offset = ctx.block().shl(I64, &slot, "3"); // arm64_32 watchOS: the object fields region begins at // `size_of::()` past the user pointer — 16 on LP64 and // padded ILP32 since #8047. Derive it from the target triple. let obj_header_size = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string(); let base = ctx.block().add(I64, &obj_handle, &obj_header_size); - let field_addr = ctx.block().add(I64, &base, &offset); - let field_ptr = ctx.block().inttoptr(I64, &field_addr); + let base_ptr = ctx.block().inttoptr(I64, &base); + // A typed GEP rather than `shl 3` + `add`: the slot is the LAST use of the + // packed word here, so with an explicit `shl` InstCombine folds + // `(packed >> 32) << 3` into `(packed >> 29) & 0x5fffffff8` and isel pays a + // 10-byte `movabs` plus an `and` for the mask. As a GEP index there is no + // `shl` to fold, so the scaled addressing mode survives — `shr $32` plus + // `(base,idx,8)`, which is what the pre-T1 tower emitted (it kept the plain + // `shr` only because its overflow block consumed the same value). + let field_ptr = ctx.block().gep(DOUBLE, &base_ptr, &[(I64, &slot)]); let val_hit = ctx.block().load(DOUBLE, &field_ptr); let val_hit_bits = ctx.block().bitcast_double_to_i64(&val_hit); let hit_deleted = ctx .block() .icmp_eq(I64, &val_hit_bits, crate::nanbox::TAG_HOLE_I64); - let deleted_idx = ctx.new_block("pic.hit.deleted"); - let deleted_label = ctx.block_label(deleted_idx); + // A hole is a field deleted since priming. It took the ordinary miss + // before (via `pic.miss`, whose way compares can never match a token the + // MRU entry still holds — `pic_prime_get` evicts a duplicate before it + // writes one), and it takes the same ordinary miss now, recording the same + // guard-fail/fallback-call pair on the way. + let hit_live_label = hit_live_idx + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| merge_label.clone()); ctx.block() .cond_br(&hit_deleted, &deleted_label, &hit_live_label); - ctx.current_block = deleted_idx; - let deleted_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); - ctx.block() - .cond_br(&deleted_cache.present, &miss_label, &cold_label); - - ctx.current_block = hit_live_idx; - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_pass", - &[(I64, &feedback_site_id)], - ); - let hit_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); - - // An object-backed Array subclass changes exact ShapeId on every numeric - // push/pop because its elements live in ordinary object slots. Its class- - // declared named prefix does not move. Runtime miss handling proves the - // complete registered prefix plus dense numeric suffix once and publishes - // a nonzero token in cache word 2 and ObjectMeta. Generic structural or - // descriptor transitions clear the object token; only the exact learned - // numeric-tail installer preserves it. - // - // Keep the ordinary miss path cheap: test the cache word first. Every - // non-Array-subclass site reads zero and leaves without touching the - // receiver's meta pointer. - ctx.current_block = prefix_guard_idx; - let cached_prefix_ptr = ctx.block().gep( - I64, - &token_cache.cache, - &[(I64, &PIC_NAMED_PREFIX_TOKEN.to_string())], - ); - let cached_prefix = ctx.block().load(I64, &cached_prefix_ptr); - let prefix_armed = ctx.block().icmp_ne(I64, &cached_prefix, "0"); - ctx.block() - .cond_br(&prefix_armed, &prefix_meta_label, &miss_label); - - // ObjectHeader::meta is the final header field: offset 8 on LP64, 12 on - // ILP32. Load it with the target pointer width, then branch before reading - // ObjectMeta so a null metadata pointer remains harmless. - ctx.current_block = prefix_meta_idx; - let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) { - 4 - } else { - 8 - }; - let meta_offset = - crate::target_layout::object_meta_slot_offset_bytes(ctx.target_triple).to_string(); - let meta_addr = ctx.block().add(I64, &obj_handle, &meta_offset); - let meta_slot = ctx.block().inttoptr(I64, &meta_addr); - let meta_load_ty = if meta_ptr_size == 4 { I32 } else { I64 }; - let meta_raw = ctx.block().load(meta_load_ty, &meta_slot); - let meta = if meta_ptr_size == 4 { - ctx.block().zext(I32, &meta_raw, I64) - } else { - meta_raw - }; - let meta_nonnull = ctx.block().icmp_ne(I64, &meta, "0"); - ctx.block() - .cond_br(&meta_nonnull, &prefix_token_label, &miss_label); - - ctx.current_block = prefix_token_idx; - let meta_ptr = ctx.block().inttoptr(I64, &meta); - // repr(C) ObjectMeta word 6. The first six u64 words are prototype, - // descriptor blooms, flags, spill, and private brand. Runtime has an - // offset assertion paired with the IR test below. - let object_prefix_ptr = ctx.block().gep(I64, &meta_ptr, &[(I64, "6")]); - let object_prefix = ctx.block().load(I64, &object_prefix_ptr); - let prefix_match = ctx.block().icmp_eq(I64, &object_prefix, &cached_prefix); - ctx.block() - .cond_br(&prefix_match, &prefix_hit_label, &miss_label); - - ctx.current_block = prefix_hit_idx; - // The exact ShapeId guard did fail, so preserve typed-feedback accounting - // just like a polymorphic-way hit: the site remains structurally - // polymorphic even though no runtime fallback call is needed. - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_fail", - &[(I64, &feedback_site_id)], - ); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_fallback_call", - &[(I64, &feedback_site_id)], - ); - let prefix_slot_ptr = ctx.block().gep(I64, &token_cache.cache, &[(I64, "1")]); - let prefix_slot = ctx.block().load(I64, &prefix_slot_ptr); - let prefix_offset = ctx.block().shl(I64, &prefix_slot, "3"); - let prefix_base = ctx.block().add(I64, &obj_handle, &obj_header_size); - let prefix_field_addr = ctx.block().add(I64, &prefix_base, &prefix_offset); - let prefix_field_ptr = ctx.block().inttoptr(I64, &prefix_field_addr); - let val_prefix = ctx.block().load(DOUBLE, &prefix_field_ptr); - let prefix_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); - - // Descriptor-bearing Array subclasses reach this duplicate of the family - // guard without ever entering `pic.token`: the exact raw-load PIC remains - // forbidden, but a runtime-proved data-only declared prefix is still safe. - // Every failure goes to the cold handler because the shape token values - // required by `pic.miss` do not dominate this path. - ctx.current_block = desc_prefix_guard_idx; - let desc_cached_prefix_ptr = ctx.block().gep( - I64, - &desc_cache.cache, - &[(I64, &PIC_NAMED_PREFIX_TOKEN.to_string())], - ); - let desc_cached_prefix = ctx.block().load(I64, &desc_cached_prefix_ptr); - let desc_prefix_armed = ctx.block().icmp_ne(I64, &desc_cached_prefix, "0"); - ctx.block() - .cond_br(&desc_prefix_armed, &desc_prefix_meta_label, &cold_label); - - ctx.current_block = desc_prefix_meta_idx; - let desc_meta_addr = ctx.block().add(I64, &obj_handle, &meta_offset); - let desc_meta_slot = ctx.block().inttoptr(I64, &desc_meta_addr); - let desc_meta_raw = ctx.block().load(meta_load_ty, &desc_meta_slot); - let desc_meta = if meta_ptr_size == 4 { - ctx.block().zext(I32, &desc_meta_raw, I64) - } else { - desc_meta_raw + let hit_end_label = match hit_live_idx { + None => ctx.block().label.clone(), + Some(idx) => { + ctx.current_block = idx; + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_guard_pass", + &[(I64, &feedback_site_id)], + ); + let label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + label + } }; - let desc_meta_nonnull = ctx.block().icmp_ne(I64, &desc_meta, "0"); - ctx.block() - .cond_br(&desc_meta_nonnull, &desc_prefix_token_label, &cold_label); - ctx.current_block = desc_prefix_token_idx; - let desc_meta_ptr = ctx.block().inttoptr(I64, &desc_meta); - let desc_object_prefix_ptr = ctx.block().gep(I64, &desc_meta_ptr, &[(I64, "6")]); - let desc_object_prefix = ctx.block().load(I64, &desc_object_prefix_ptr); - let desc_prefix_match = ctx - .block() - .icmp_eq(I64, &desc_object_prefix, &desc_cached_prefix); - ctx.block() - .cond_br(&desc_prefix_match, &desc_prefix_hit_label, &cold_label); - - ctx.current_block = desc_prefix_hit_idx; - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_fail", - &[(I64, &feedback_site_id)], - ); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_fallback_call", - &[(I64, &feedback_site_id)], - ); - let desc_prefix_slot_ptr = ctx.block().gep(I64, &desc_cache.cache, &[(I64, "1")]); - let desc_prefix_slot = ctx.block().load(I64, &desc_prefix_slot_ptr); - let desc_prefix_offset = ctx.block().shl(I64, &desc_prefix_slot, "3"); - let desc_prefix_base = ctx.block().add(I64, &obj_handle, &obj_header_size); - let desc_prefix_field_addr = ctx.block().add(I64, &desc_prefix_base, &desc_prefix_offset); - let desc_prefix_field_ptr = ctx.block().inttoptr(I64, &desc_prefix_field_addr); - let val_desc_prefix = ctx.block().load(DOUBLE, &desc_prefix_field_ptr); - let desc_prefix_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); + // The hit's hole lands here rather than on the shared exit — see the + // congruence note where the block is minted. + ctx.current_block = deleted_idx; + ctx.block().br(&cold_label); // PIC miss on the MRU entry — before paying for the call, try the // polymorphic ways (#7753). // - // `js_object_get_field_ic_miss` is not a cheap fallback: it re-derives the - // receiver kind from scratch (proxy band, closure magic, registered-buffer - // and typed-array registries, small-handle dispatch), reads the + // The slow entry is not a cheap fallback: it re-derives the receiver kind + // from scratch (proxy band, closure magic, registered-buffer and + // typed-array registries, small-handle dispatch), reads the // accessors-in-use thread-local, then linear-scans the keys array with a // `js_string_equals` per key. On a site whose receiver alternates between a // handful of shapes — the shape of every discriminated-union dispatch — @@ -781,9 +657,10 @@ pub(crate) fn lower_generic_property_get( // // # Why this block is DOMINATED by `pic.token` (#7907) // - // Its only predecessor is `pic.token` after the MRU token did not match. - // The exact descriptor identity proves cached-slot bounds, so `token` - // is everything the way compares need. + // Its only predecessor is `pic.token.miss`, which is `pic.token`'s. The + // exact descriptor identity proves cached-slot bounds, so `token` is + // everything the way compares need, and the cache pointer arrives on one + // edge rather than through a phi. // // #7883 could not rely on that: it routed the two receiver-validation // failures here as well, which left the values live on only some edges, so @@ -795,24 +672,8 @@ pub(crate) fn lower_generic_property_get( // the duplicate ladder sat on the hot path. Measured on `interp.ts`'s // `evalNode`, the single hottest instruction in the whole program was the // redundant receiver reconstruction inside this block. - // - // Sending the two validation failures to `pic.miss.cold` instead is what - // establishes the dominance. Nothing about the predicate changed: a - // receiver that fails either check also fails `way_hit` (which ANDs - // `is_object` in), so it could never have resolved a way — the compares - // were dead work for it. ctx.current_block = miss_idx; - // All incoming cache values have passed their own presence guard. The - // merge preserves that proof without resolving the full cache on a hit. - let cache_ref = ctx.block().phi( - PTR, - &[ - (&token_cache.cache, &prefix_guard_label), - (&token_cache.cache, &prefix_meta_label), - (&token_cache.cache, &prefix_token_label), - (&deleted_cache.cache, &deleted_label), - ], - ); + let cache_ref = token_cache.cache.clone(); crate::expr::emit_typed_feedback_record_call( ctx.block(), "js_typed_feedback_record_guard_fail", @@ -915,33 +776,36 @@ pub(crate) fn lower_generic_property_get( let way_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); - // #7907: receiver-validation failure. `way_hit` requires a real pointer to - // a plain descriptor-free `ObjectHeader`, so a receiver that got here can - // never match a way — it goes straight to the handler, which reproduces the - // whole ladder anyway (proxy band, closure magic, buffer/typed-array - // registries, small-handle dispatch). The typed-feedback counters are the - // same two `pic.miss` records on the same edges, so the feedback signal is - // byte-identical to what the merged block reported. - ctx.current_block = cold_idx; - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_fail", - &[(I64, &feedback_site_id)], - ); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_fallback_call", - &[(I64, &feedback_site_id)], - ); - ctx.block().br(&call_label); + // #7907: receiver-validation failure. A receiver that gets here can never + // match a way — the compares require a real pointer to a plain + // descriptor-free `ObjectHeader` — so it goes straight to the handler, + // which reproduces the whole ladder anyway (proxy band, closure magic, + // buffer/typed-array registries, small-handle dispatch). The typed-feedback + // counters are the same two records on the same edges, so the feedback + // signal is byte-identical to what the pre-T1 blocks reported. + if let Some(cold_idx) = cold_idx { + ctx.current_block = cold_idx; + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_guard_fail", + &[(I64, &feedback_site_id)], + ); + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_fallback_call", + &[(I64, &feedback_site_id)], + ); + ctx.block().br(&call_label); + } - // PIC miss: slow path with cache population. + // The object exit: one call reproducing every pointer-path arm this tower + // used to expand. ctx.current_block = call_idx; crate::expr::emit_versioned_loop_callback_deopt(ctx); let miss_key_handle = emit_key_handle(ctx, &key_handle_global); let val_miss = ctx.block().call( DOUBLE, - "js_object_get_field_ic_miss_packed", + "js_object_get_field_ic_slow", &[ (I64, &obj_handle), (I64, &miss_key_handle), @@ -952,149 +816,43 @@ pub(crate) fn lower_generic_property_get( let miss_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); - // Merge PIC hit + way hit + miss, then jump to the outer recv-valid merge. - ctx.current_block = merge_idx; - let pic_val = ctx.block().phi( - DOUBLE, - &[ - (&val_hit, &hit_end_label), - (&val_ovf, &ovf_end_label), - (&val_prefix, &prefix_end_label), - (&val_desc_prefix, &desc_prefix_end_label), - (&val_way, &way_end_label), - (&val_miss, &miss_end_label), - ], - ); - let pic_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); - // Native Map/Set `.size`: their common leading field was admitted only by // the exact live GC-kind checks above. Keep the read inline; calling // `js_map_size` / `js_set_size` would reclassify the same receiver again. - let collection_size_arm = if let Some(collection_idx) = collection_size_idx { + let collection_size_arm = collection_size_idx.map(|collection_idx| { ctx.current_block = collection_idx; let size_i32 = ctx.block().safe_load_i32_from_ptr(&obj_handle); let size = ctx.block().uitofp(I32, &size_i32, DOUBLE); let collection_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); - Some((size, collection_end_label)) - } else { - None - }; - - // Invalid receiver: per JS spec, `undefined` and `null` - // throw a TypeError; other non-pointer tags (int32, bool, - // plain f64, bigint) should auto-box and look up via the - // primitive's prototype. Perry doesn't implement primitive - // auto-boxing yet, so non-nullish primitives continue to - // return `undefined` to preserve existing behavior. - // - // Issue #462: bare `obj.foo` against TAG_UNDEFINED / - // TAG_NULL silently returned undefined, which masked - // unimplemented-API bugs (e.g. `crypto.subtle.encrypt(...)` - // ran to completion as a chain of no-ops). Funnel the - // nullish receiver into the runtime helper which prints a - // node-shaped diagnostic and aborts. - ctx.current_block = invalid_idx; - let is_undef = ctx - .block() - .icmp_eq(I64, &obj_bits, crate::nanbox::TAG_UNDEFINED_I64); - let is_null = ctx - .block() - .icmp_eq(I64, &obj_bits, crate::nanbox::TAG_NULL_I64); - let is_nullish = ctx.block().or(I1, &is_undef, &is_null); - let throw_idx = ctx.new_block("pget.throw_nullish"); - let undef_idx = ctx.new_block("pget.recv_undef_return"); - let throw_label = ctx.block_label(throw_idx); - let undef_label = ctx.block_label(undef_idx); - ctx.block().cond_br(&is_nullish, &throw_label, &undef_label); - - // Throw path: helper aborts the process; block ends with - // `unreachable` because the helper's `-> !` return is - // not visible to LLVM. - ctx.current_block = throw_idx; - let prop_entry = ctx.strings.entry(key_idx); - let prop_bytes_global = format!("@{}", prop_entry.bytes_global); - let prop_len_str = prop_entry.byte_len.to_string(); - let is_null_i32 = ctx.block().zext(I1, &is_null, I32); - ctx.block().call_void( - "js_throw_type_error_property_access", - &[ - (I32, &is_null_i32), - (PTR, &prop_bytes_global), - (I64, &prop_len_str), - ], - ); - ctx.block().unreachable(); - - // Undef-return path: existing fall-through for non-nullish - // invalid receivers. Route through the runtime helper first - // so non-pointer typed shapes can still report a sensible - // value when the runtime knows what they are. Today this - // unblocks Date `.constructor` (Date stores as a raw f64 - // timestamp, so the codegen receiver-tag check at line ~4212 - // rejects it as non-pointer — yet the runtime's - // `js_object_get_field_by_name_f64` recognizes the bit - // pattern via `DATE_REGISTRY` and returns the global Date - // constructor closure). Date-fns `constructFrom` blocker. - ctx.current_block = undef_idx; - let undef_key_handle = emit_key_handle(ctx, &key_handle_global); - let undef_val = ctx.block().call( - DOUBLE, - "js_object_get_field_by_name_f64", - &[(I64, &obj_bits), (I64, &undef_key_handle)], - ); - let invalid_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); - - // SSO receiver: dispatch directly to the runtime by-name - // helper, which reads `.length` inline from the NaN-box - // payload and returns `undefined` for other keys. Bypasses - // the PIC entirely (PIC would read garbage memory). The - // key handle has already been extracted above. - ctx.current_block = sso_idx; - let sso_val = if inline_string_length { - // `.length` of an SSO string is the length byte in bits 40..47 of the - // NaN-box itself — the same extract `js_object_get_field_by_name_f64` - // performs, minus the call and the key decode. - let len_shifted = ctx.block().lshr(I64, &obj_bits, "40"); - let len_byte = ctx.block().and(I64, &len_shifted, "255"); - ctx.block().uitofp(I64, &len_byte, DOUBLE) - } else { - let sso_key_handle = emit_key_handle(ctx, &key_handle_global); - ctx.block().call( - DOUBLE, - "js_object_get_field_by_name_f64", - &[(I64, &obj_bits), (I64, &sso_key_handle)], - ) - }; - let sso_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); + ctx.block().br(&merge_label); + (size, collection_end_label) + }); // Heap string `.length`: `utf16_len` is the leading `u32` of // `StringHeader` — the identical load the proven-string lowering in // `property_get.rs` emits (`strlen.heap`). `safe_load_i32_from_ptr` // keeps a sub-page handle off the load. - let strlen_heap_arm = if let Some(heap_idx) = strlen_heap_idx { + let strlen_heap_arm = strlen_heap_idx.map(|heap_idx| { ctx.current_block = heap_idx; let len_i32 = ctx.block().safe_load_i32_from_ptr(&obj_handle); let heap_len = ctx.block().uitofp(I32, &len_i32, DOUBLE); let heap_end_label = ctx.block().label.clone(); - ctx.block().br(&final_merge_label); - Some((heap_len, heap_end_label)) - } else { - None - }; + ctx.block().br(&merge_label); + (heap_len, heap_end_label) + }); - // Outer merge joins PIC result + invalid-receiver undefined - // + SSO result + class-ref dispatch result (+ heap-string `.length`). - ctx.current_block = final_merge_idx; + // One merge for the whole tower: the inline hit, a way hit, the two slow + // calls, and whichever short-circuit arms this key grew. + ctx.current_block = merge_idx; let mut incoming: Vec<(&str, &str)> = vec![ - (&pic_val, &pic_end_label), - (&undef_val, &invalid_end_label), - (&sso_val, &sso_end_label), - (&class_ref_result, &class_ref_end_label), + (&val_hit, &hit_end_label), + (&val_way, &way_end_label), + (&val_miss, &miss_end_label), + (&val_nonptr, &nonptr_end_label), ]; + if let Some((sso_val, sso_end_label)) = sso_arm.as_ref() { + incoming.push((sso_val, sso_end_label)); + } if let Some((heap_len, heap_end_label)) = strlen_heap_arm.as_ref() { incoming.push((heap_len, heap_end_label)); } diff --git a/crates/perry-codegen/src/expr/property_get/helpers.rs b/crates/perry-codegen/src/expr/property_get/helpers.rs index 51a8e8dbe6..302fedd4b8 100644 --- a/crates/perry-codegen/src/expr/property_get/helpers.rs +++ b/crates/perry-codegen/src/expr/property_get/helpers.rs @@ -368,29 +368,14 @@ pub(crate) fn lower_raw_f64_class_field_get_for_number_context( // array; the report below must not re-derive it from the expression // shape, which the binding form does not carry. let arr_id = fact.array_local_id; - // The counter's canonical i32 slot is what the matcher required for - // every index form that reads the counter; without it there is nothing - // to index with. A constant index reads no counter and needs none. - let counter_slot = ctx.i32_counter_slots.get(&fact.index_local_id).cloned(); - if counter_slot.is_some() || !fact.index.needs_counter_i32_slot() { - // #10123: the index the preheader discharged a bounds obligation - // for. All three forms are i32 and in `[0, length)` by the time - // they reach the GEP, which is why the fast clone pays no per-read - // bounds test in any of them. - let idx_i32 = match &fact.index { - crate::expr::ElementShapeIndex::Counter => { - let slot = counter_slot.expect("checked above"); - ctx.block().load(I32, &slot) - } - crate::expr::ElementShapeIndex::Constant(k) => k.to_string(), - // The derived `const d = j % m` binding's own slot, written by - // the `Let` arm in `stmt/let_stmt.rs` earlier in this same - // iteration. - crate::expr::ElementShapeIndex::DerivedMod { slot, .. } => { - let slot = slot.clone(); - ctx.block().load(I32, &slot) - } - }; + // #10123: the index the preheader discharged a bounds obligation for. + // Every form is an i32 in `[0, length)` by the time it reaches the GEP, + // which is why the fast clone pays no per-read bounds test in any of + // them. `None` means the counter's canonical i32 slot the matcher + // required is missing, so there is nothing to index with. + if let Some(idx_i32) = + crate::expr::element_shape_guard::emit_element_shape_index(ctx, &fact) + { let value = crate::expr::element_shape_guard::emit_element_shape_field_load( ctx, &fact, diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 59f23e97e4..e9a3b0f654 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -136,10 +136,15 @@ fn imported_variable_read_preserves_class_tags_and_calls_the_live_getter_once() (operand == format!("{value} to i64")).then_some(result) }) .expect("getter result must be classified by its intact value tag"); + // #9366's invariant, one indirection later: T1 moved the INT32 class-ref + // arm (and its `js_typed_feedback_object_get_field_by_name_f64` call) into + // `js_object_get_field_ic_nonptr`, which routes on the tag — so the bits it + // receives must still be the getter's UNMASKED value. A masked handle here + // would lose the 0x7FFE tag and the class would dispatch as an object. assert!( ir.lines().any(|line| { - line.contains("call double @js_typed_feedback_object_get_field_by_name_f64(") - && line.contains(&format!(", i64 {bits},")) + line.contains("call double @js_object_get_field_ic_nonptr(") + && line.contains(&format!("i64 {bits},")) }), "class dispatch must receive the getter's unmasked value bits:\n{ir}" ); @@ -391,40 +396,56 @@ fn pic_cache_layout_matches_runtime() { ); } -/// Object-backed Array subclasses mint one ShapeId per numeric tail length. -/// A named-field site on such a receiver must try the independently proved -/// class prefix before falling into the bounded shape ways / runtime miss. +/// Object-backed Array subclasses mint one ShapeId per numeric tail length, so +/// a named-field site on such a receiver is served from the independently +/// proved class prefix rather than the exact ShapeId. +/// +/// Renamed from `generic_property_get_emits_array_subclass_named_prefix_guard` +/// (T1): the proof itself is unchanged and still runs on exactly the same two +/// words, but it runs in `js_object_get_field_ic_slow` instead of in FOUR +/// emitted blocks per site (`pic.prefix.*`) plus FOUR more for the +/// descriptor-bearing twin (`pic.desc.prefix.*`). Its behaviour is pinned by +/// `an_armed_named_prefix_serves_the_cached_slot` in +/// `perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs`; what this test +/// keeps is the CODEGEN half of the contract — the emitted site must hand the +/// runtime the two operands that proof reads, and must not have grown its own +/// copy back. #[test] -fn generic_property_get_emits_array_subclass_named_prefix_guard() { - use crate::expr::property_get::generic_dispatch::PIC_NAMED_PREFIX_TOKEN; - +fn array_subclass_named_prefix_proof_is_reached_through_the_one_exit() { let ir = emit(false, None); - let guard = ir - .find("\npic.prefix.guard") - .unwrap_or_else(|| panic!("expected a named-prefix guard block:\n{ir}")); - let token = ir - .find("\npic.prefix.token") - .unwrap_or_else(|| panic!("expected a named-prefix token block:\n{ir}")); - let hit = ir - .find("\npic.prefix.hit") - .unwrap_or_else(|| panic!("expected a named-prefix hit block:\n{ir}")); - let miss = ir - .find("\npic.miss") - .unwrap_or_else(|| panic!("expected the ordinary PIC miss block:\n{ir}")); - assert!( - guard < token && token < hit && hit < miss, - "prefix guard must precede the ordinary miss path:\n{ir}" - ); - - let guard_body = &ir[guard..token]; - assert!( - guard_body.contains(&format!("i64 {PIC_NAMED_PREFIX_TOKEN}\n")), - "the cheap first gate must read cache word 2 before touching ObjectMeta:\n{guard_body}" - ); - let token_body = &ir[token..hit]; + for gone in [ + "pic.prefix.guard", + "pic.prefix.meta", + "pic.prefix.token", + "pic.prefix.hit", + "pic.desc.classify", + "pic.desc.prefix.guard", + "pic.desc.prefix.meta", + "pic.desc.prefix.token", + "pic.desc.prefix.hit", + ] { + assert!( + !ir.contains(gone), + "the named-prefix ladder must not be emitted per site any more, \ + found `{gone}`:\n{ir}" + ); + } + // The runtime half reads cache word 2 against ObjectMeta word 6 and then + // the cached slot, so the emitted site has to hand it both per-site + // globals — a call that lost either operand would silently stop serving + // Array-subclass named fields and fall back to the full lookup. + let call = ir + .find("\npic.miss.call") + .unwrap_or_else(|| panic!("expected the single slow-exit block:\n{ir}")); + let call_line = ir[call..] + .lines() + .find(|l| l.contains("@js_object_get_field_ic_slow(")) + .unwrap_or_else(|| panic!("expected the one slow call:\n{ir}")); assert!( - token_body.contains("getelementptr i64") && token_body.contains("i64 6\n"), - "ObjectMeta word 6 must carry the runtime-paired prefix token:\n{token_body}" + call_line.contains("ptr @perry_ic_") && call_line.contains("_packed_get"), + "the slow exit must receive BOTH the cache slot (which holds the \ + named-prefix token in word 2) and the packed MRU word, or the runtime \ + cannot reproduce the arms this site stopped emitting:\n{call_line}" ); } @@ -471,8 +492,8 @@ fn generic_property_get_tries_ways_before_calling_the_miss_handler() { miss call — otherwise the compares are not gating anything:\n{ways_body}" ); assert!( - !ways_body.contains("call double @js_object_get_field_ic_miss"), - "the miss call must not sit inside the way block:\n{ways_body}" + !ways_body.contains("call double @js_object_get_field_ic"), + "the slow call must not sit inside the way block:\n{ways_body}" ); // The way compares read (token, slot) pairs at words PIC_WAY_BASE.. and the // gate reads the state word — all inside pic.ways, none anywhere else. @@ -520,10 +541,43 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { main.contains("@perry_ic_"), "test premise: the generic read reaches the inline PIC:\n{ir}" ); + // T1: the landing block is now the single slow exit itself, and the + // dominance is structural — `pic.miss` has exactly ONE predecessor, + // `pic.token.miss`, which `pic.token` dominates. Assert that directly: + // routing any receiver-validation failure back into `pic.miss` would add a + // predecessor and immediately re-introduce the phis #7907 removed. + // `pic.miss` carries a numeric suffix and `pic.miss.call` starts with the + // same text, so match the block's own label exactly and then count the + // branches whose TARGET is that label (a `br i1` naming both blocks counts + // once, for the right one). + let miss_label = main + .lines() + .filter(|l| !l.starts_with(' ') && l.ends_with(':')) + .map(|l| l.trim_end_matches(':')) + .find(|l| { + l.strip_prefix("pic.miss.") + .is_some_and(|tail| tail.chars().all(|c| c.is_ascii_digit())) + }) + .unwrap_or_else(|| panic!("expected a pic.miss block:\n{ir}")) + .to_string(); + let preds = main + .lines() + .filter(|l| l.trim_start().starts_with("br ")) + .filter(|l| { + l.split("label %") + .skip(1) + .any(|t| t.trim_end_matches(&[',', ' '][..]) == miss_label) + }) + .count(); + assert_eq!( + preds, 1, + "pic.miss must have exactly one predecessor (pic.token.miss), or it is \ + no longer dominated by pic.token:\n{ir}" + ); assert!( - main.contains("\npic.miss.cold"), - "the two receiver-validation failures need their own landing block, \ - otherwise pic.miss is not dominated by pic.token:\n{ir}" + main.contains("label %pic.miss.call"), + "every receiver-validation failure must land on the single slow \ + exit:\n{ir}" ); assert!( !main.contains("@PERRY_IC_EPOCH"), @@ -963,28 +1017,56 @@ fn generic_length_read_serves_a_string_inline() { ); // Everything that is NOT a string keeps the tower. assert!( - ir.contains("@perry_ic_") && ir.contains("js_object_get_field_ic_miss"), - "non-string receivers must still reach the inline PIC and its miss \ - handler:\n{ir}" + ir.contains("@perry_ic_") && ir.contains("js_object_get_field_ic_slow"), + "non-string receivers must still reach the inline PIC and its slow \ + exit:\n{ir}" ); } /// The short-circuit is keyed on the property name: any other key on a string -/// receiver (`s.charCodeAt`, `s.constructor`) still needs the runtime, so no -/// other read may grow the string blocks. +/// receiver (`s.charCodeAt`, `s.constructor`) still needs the runtime. +/// +/// T1 moved the SSO arm itself behind the one exit — an SSO receiver with any +/// key but `length` is served by `js_object_get_field_ic_slow`'s tag ladder +/// (`sso_receiver_routes_to_the_by_name_helper` in +/// `ic_miss/ic_slow.rs` pins that it still reaches the by-name helper). What +/// codegen must guarantee is that such a receiver LEAVES: it must never fall +/// into the PIC, whose header loads would read the SSO payload as an address. #[test] fn generic_non_length_read_keeps_the_whole_tower() { let ir = emit_read("charCodeAt"); + for gone in ["pget.strlen_heap", "pget.recv_sso"] { + assert!( + !ir.contains(gone), + "only `.length` may grow an inline string arm, found `{gone}`:\n{ir}" + ); + } + // 32765 = (STRING_TAG|POINTER_TAG) & 0xFFFD: the one test that decides + // whether the receiver may be dereferenced at all. Its false edge must be + // the NON-POINTER exit — a distinct block with a distinct callee, which is + // what stops SimplifyCFG folding this guard into the next one. + let tag_branch = ir + .lines() + .find(|l| l.contains("icmp eq i64") && l.contains("32765")) + .unwrap_or_else(|| panic!("expected the receiver-tag test:\n{ir}")); + let cond = tag_branch + .trim() + .split_once(" = ") + .map(|(lhs, _)| lhs.to_string()) + .unwrap_or_else(|| panic!("malformed tag test: {tag_branch}")); + let branch = ir + .lines() + .find(|l| l.trim_start().starts_with(&format!("br i1 {cond},"))) + .unwrap_or_else(|| panic!("expected a branch on the receiver tag:\n{ir}")); assert!( - !ir.contains("pget.strlen_heap"), - "only `.length` may take the inline string arm:\n{ir}" + branch.contains("label %pget.recv_other") && !branch.contains("label %pic.miss.call"), + "a non-pointer receiver must leave for its OWN exit — sharing the \ + object exit's block is what cost +4 instructions per hit:\n{branch}" ); - let sso = ir - .find("\npget.recv_sso") - .unwrap_or_else(|| panic!("expected an SSO receiver block:\n{ir}")); assert!( - ir[sso..].contains("js_object_get_field_by_name_f64"), - "a non-`length` SSO read must still call the by-name helper:\n{ir}" + ir.contains("@js_object_get_field_ic_slow(") + && ir.contains("@js_object_get_field_ic_nonptr("), + "the tower must still reach both slow entries:\n{ir}" ); } @@ -1015,7 +1097,7 @@ fn generic_size_read_serves_native_collections_inline() { {collection_body}" ); assert!( - ir.contains("@perry_ic_") && ir.contains("js_object_get_field_ic_miss"), + ir.contains("@perry_ic_") && ir.contains("js_object_get_field_ic_slow"), "non-collection receivers must retain the generic property tower:\n{ir}" ); } @@ -1069,10 +1151,18 @@ fn packed_pic_header_guard_is_endianness_aware() { ir.contains("load i16"), "descriptor guard must retain native endianness: {ir}" ); + assert!( + ir.contains(", 2048"), + "the native-endian arm must still test OBJ_FLAG_HAS_DESCRIPTORS: {ir}" + ); } + // T1: the descriptor-bearing fallback is the one exit. It must still + // be a distinct EDGE — a descriptor-bearing receiver may never take + // the raw-slot hit — and the runtime keeps the Array-subclass + // named-prefix exception behind it. assert!( - ir.contains("pic.desc.prefix.guard"), - "descriptor fallback must remain: {ir}" + ir.contains("@js_object_get_field_ic_slow(") && ir.contains("\npic.miss.call"), + "descriptor fallback must remain, through the one exit: {ir}" ); } } @@ -1088,7 +1178,7 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { ir.contains("load atomic i64") && ir.contains("monotonic, align 8"), "{ir}" ); - assert!(ir.contains("@js_object_get_field_ic_miss_packed("), "{ir}"); + assert!(ir.contains("@js_object_get_field_ic_slow("), "{ir}"); assert!( ir.contains("trunc i64") && ir.contains("icmp ne i64"), "{ir}" @@ -1098,3 +1188,104 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { "a full-cache dereference must still guard a null site: {ir}" ); } + +/// T1: the whole point — per untyped `obj.prop` the emitted tower is TWO calls +/// and a handful of blocks, with the inline hit and the polymorphic ways kept. +/// +/// This is a ratchet, so it is an EXACT count in both dimensions. The tower it +/// replaced expanded 33 tower blocks and SIX runtime call sites per site +/// (`js_object_get_field_by_name_f64` twice, the feedback-wrapped class-ref +/// helper, `js_throw_type_error_property_access`, +/// `js_object_get_field_ic_overflow_load`, `js_object_get_field_ic_miss_packed`), +/// each one a statepoint whose live GC values are written into +/// `.perry_gcmap`. On @babel/parser that was 29% of all emitted IR over 6,487 +/// sites; a single arm creeping back inline is a regression measured in +/// megabytes of `.text`, and nothing else in the suite would report it. +/// +/// Two and not one: a single shared exit let SimplifyCFG fold the receiver-tag +/// test and the small-handle test into one flat predicate, costing +4.00 +/// instructions on every HIT (measured). The separate non-pointer callee is +/// what keeps that guard chain branchy, so the count below is 2 — and a change +/// that makes it 1 is a hit-path regression, not a size win. +#[test] +fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { + let ir = emit(false, None); + let func = ir + .split("\ndefine ") + .find(|f| f.contains("\npic.miss.call")) + .unwrap_or_else(|| panic!("no function contains the generic tower:\n{ir}")); + + // Every call/invoke in the whole function, by callee. Feedback records are + // compile-time gated and absent from this build; anything else must be the + // one exit (the fixture's module init contributes its own calls, so match + // on the property-GET family rather than on a total). + let pget_calls: Vec<&str> = func + .lines() + .filter(|l| l.contains(" call ") || l.contains(" invoke ")) + .filter_map(|l| l.split(" @").nth(1)) + .filter_map(|c| c.split('(').next()) + .filter(|c| { + c.starts_with("js_object_get_field") + || c.starts_with("js_typed_feedback_object_get_field") + || *c == "js_throw_type_error_property_access" + }) + .collect(); + let mut sorted = pget_calls.clone(); + sorted.sort(); + assert_eq!( + sorted, + vec![ + "js_object_get_field_ic_nonptr", + "js_object_get_field_ic_slow" + ], + "the tower must expand exactly two property-GET call sites:\n{func}" + ); + + let blocks: Vec<&str> = func + .lines() + .filter(|l| !l.starts_with(' ') && l.ends_with(':')) + .map(|l| l.trim_end_matches(':')) + .filter(|l| l.starts_with("pget.") || l.starts_with("pic.")) + .collect(); + let mut expected = vec![ + // the guard chain and the inline hit + "pget.recv_ok", + // the non-pointer exit, off the tag test's false edge + "pget.recv_other", + "pic.recv_hdr", + "pic.token", + "pic.token.miss", + "pic.hit", + "pic.hit.inline", + // The hit's hole edge keeps its own landing block so its tail is not + // congruent with `pic.way.load`'s; `pic.hit.live` exists only when + // typed feedback has something to record on the live edge. + "pic.hit.deleted", + // the polymorphic ways, deliberately still inline (#7753) + "pic.miss", + "pic.ways", + "pic.way.load", + "pic.way.live", + // the one exit, and the join + "pic.miss.call", + "pget.recv_merge", + ]; + // Labels carry a numeric suffix (`pic.ways.16`); strip it for comparison. + let mut normalized: Vec = blocks + .iter() + .map(|b| { + let mut parts: Vec<&str> = b.split('.').collect(); + if parts.last().is_some_and(|p| p.parse::().is_ok()) { + parts.pop(); + } + parts.join(".") + }) + .collect(); + normalized.sort(); + expected.sort(); + assert_eq!( + normalized, + expected.iter().map(|s| s.to_string()).collect::>(), + "the emitted tower's block set changed:\n{func}" + ); +} diff --git a/crates/perry-codegen/src/expr/shadow_slot.rs b/crates/perry-codegen/src/expr/shadow_slot.rs index 4f1ff8e459..2383e3890d 100644 --- a/crates/perry-codegen/src/expr/shadow_slot.rs +++ b/crates/perry-codegen/src/expr/shadow_slot.rs @@ -176,18 +176,32 @@ pub(crate) fn emit_shadow_slot_clear(ctx: &mut FnCtx<'_>, slot_idx: u32) { // #10123's derived index (`const d = j % m`) is the same case for the same // reason: its `Let` emits one `srem` into a private i32 alloca, never a // shadow bind, so a lexical-death clear would be the clone's only call. + // + // #10185's carried index (`c = (a*c + b) % m`) and its optional + // `const index = c` alias are two more of the same. if ctx.element_shape_loop_facts.iter().any(|fact| { - let virtual_binding = match &fact.index { - crate::expr::ElementShapeIndex::DerivedMod { local_id, .. } => Some(*local_id), - _ => None, - }; fact.element_binding .into_iter() - .chain(virtual_binding) + .chain(fact.index.virtual_locals()) .any(|id| ctx.shadow_slot_map.get(&id) == Some(&slot_idx)) }) { return; } + // #10185: a SYNTHESIZED fast-clone body (the accumulator fold, the carried + // write-back) no longer has the statement indices the function-wide clear + // map is keyed by, so every clear reached from inside it would be + // attributed to some other statement's local. Suppress them for the fast + // clone only: it is call-free, so no collection can observe a shadow slot + // while it runs, an over-rooted stale value is rewritten like any root, and + // the slow clone — lowered from the ORIGINAL body after the facts are + // popped — keeps every clear it always had. + if ctx + .element_shape_loop_facts + .iter() + .any(|fact| fact.synthesized_body) + { + return; + } // Never-bound slot: it provably still holds its initial 0 (slots are only // written through bind/set, and every value-set site binds first), so the // clear would be a redundant `js_shadow_slot_set(idx, 0)` TLS hit. diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index e1e8d17fe4..265da914e7 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -750,6 +750,11 @@ mod tests { "js_rel_lt", "js_rel_gt", "js_object_get_field_ic_miss_packed", + // T1: the generic-get tower's two slow exits. One reaches the same + // `get_field_ic_miss_impl`, the other the by-name helper, so both + // allocate and can run a user getter. + "js_object_get_field_ic_slow", + "js_object_get_field_ic_nonptr", ] { assert_ne!( classify_direct_callee(name), diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs index 531516b8ef..823f7525e9 100644 --- a/crates/perry-codegen/src/module/linkage.rs +++ b/crates/perry-codegen/src/module/linkage.rs @@ -251,7 +251,10 @@ pub(crate) fn helper_decl_attrs(name: &str) -> &'static str { // register saves and code layout focused on the inline continuation. // `cold` is only a profitability hint: both calls remain fully // memory-clobbering and the cache miss remains GC-capable/throwing. - "js_object_get_field_ic_miss_packed" | "js_write_barrier_root_nanbox" => " cold", + "js_object_get_field_ic_miss_packed" + | "js_object_get_field_ic_slow" + | "js_object_get_field_ic_nonptr" + | "js_write_barrier_root_nanbox" => " cold", // PURE — each verified: pure bit tests/masking on the f64/i64 args, // total over arbitrary bits, no memory access anywhere in the body. // js_nanbox_pointer value/nanbox.rs — tag ladder, 0 → TAG_NULL diff --git a/crates/perry-codegen/src/native_value/verify/raw_f64.rs b/crates/perry-codegen/src/native_value/verify/raw_f64.rs index 51d2644550..0e15791933 100644 --- a/crates/perry-codegen/src/native_value/verify/raw_f64.rs +++ b/crates/perry-codegen/src/native_value/verify/raw_f64.rs @@ -96,6 +96,13 @@ pub(crate) fn raw_f64_dynamic_fallback_record(record: &NativeRepRecord) -> bool | ("PackedI32LoopGuard", "packed_i32_loop_fallback") | ("PackedU32LoopGuard", "packed_u32_loop_fallback") | ("ClassFieldGet", "js_object_get_field_by_name_f64") + // T3 one-exit class-field GET: the inline tower's miss arm is a + // single `js_class_field_get_ic` call instead of the nullish + // diamond + by-name lookup, so the dynamic-fallback record now + // names the IC. The by-name pair stays: it is still the contract + // asserted by `verify/tests.rs`, and dropping an entry rather + // than adding to it is exactly how #8858 silently lost one. + | ("ClassFieldGet", "js_class_field_get_ic") | ("ClassFieldSet", "js_object_set_field_by_name") ) } diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index e909ba79a7..f35aa1b628 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -366,6 +366,17 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // single call for oversized modules. Args: (obj_bits, key_handle, site_id, // per-site IC cache global) -> field value. module.declare_function("js_object_get_field_ic", DOUBLE, &[I64, I64, I64, PTR]); + // T1: the two exits of the inline generic-get tower. Every guard failure — + // SSO / INT32 class ref / nullish / non-object receiver / overflow slot / + // deleted slot / named prefix / miss+prime — branches to one of these + // instead of expanding its own arm and its own call. + // + // Non-pointer receiver (the emitted tag test failed): (obj_bits, key_handle, + // site_id) -> field value. No cache: none of its arms can prime one. + module.declare_function("js_object_get_field_ic_nonptr", DOUBLE, &[I64, I64, I64]); + // Heap-pointer receiver: (masked obj_handle, key_handle, per-site IC cache + // SLOT, per-site packed MRU word) -> field value. + module.declare_function("js_object_get_field_ic_slow", DOUBLE, &[I64, I64, PTR, PTR]); // Object rest destructuring: copy all properties from src except excluded keys. // Takes a src object ptr and an array of NaN-boxed strings (the excluded keys), // returns a new object pointer. diff --git a/crates/perry-codegen/src/stmt/cached_field_index_return.rs b/crates/perry-codegen/src/stmt/cached_field_index_return.rs index 30339319ab..07759705a5 100644 --- a/crates/perry-codegen/src/stmt/cached_field_index_return.rs +++ b/crates/perry-codegen/src/stmt/cached_field_index_return.rs @@ -207,7 +207,11 @@ pub(super) fn try_emit_cached_field_index_return( .cond_br(&exact, &field_load_label, &prefix_meta_label); ctx.current_block = prefix_meta_idx; - let cached_prefix_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); + // The same cache word the generic property-get tower's named-prefix proof + // uses, and the same one the runtime publishes it in. + let prefix_word = + crate::expr::property_get::generic_dispatch::PIC_NAMED_PREFIX_TOKEN.to_string(); + let cached_prefix_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, &prefix_word)]); let cached_prefix = ctx.block().load(I64, &cached_prefix_ptr); let prefix_armed = ctx.block().icmp_ne(I64, &cached_prefix, "0"); let pointer_bytes = if crate::target_layout::target_is_ilp32(ctx.target_triple) { diff --git a/crates/perry-codegen/src/stmt/element_shape_carried.rs b/crates/perry-codegen/src/stmt/element_shape_carried.rs new file mode 100644 index 0000000000..3924d8e733 --- /dev/null +++ b/crates/perry-codegen/src/stmt/element_shape_carried.rs @@ -0,0 +1,416 @@ +//! #10185: the element-shape clone's LOOP-CARRIED index — the `random` access +//! shape. +//! +//! ```text +//! let cursor = 0; +//! for (let i = 0; i < count; i++) { +//! cursor = (cursor * 17 + 7) % length; // the recurrence +//! const index = cursor; // optional alias +//! sum += rows[index].id; +//! } +//! ``` +//! +//! ## What makes this different from `const d = i % m` +//! +//! `d` is dead the moment the iteration ends, so #10123 could keep it entirely +//! virtual — its `Let` writes a private i32 alloca and nothing ever reads the +//! real slot. `cursor` is a real `let` that OUTLIVES the loop, so the clone +//! owes a write-back. +//! +//! **Where that write-back goes is a correctness question, not a scheduling +//! one.** The clone's residual check side-exits by resuming the CURRENT +//! iteration in the slow clone, which re-runs the whole body — the recurrence +//! included. A write-back at the update site would therefore apply +//! `cursor = (cursor * 17 + 7) % length` twice for that iteration and every +//! subsequent index would be wrong (silently: still in bounds, still a valid +//! record, just not the one JavaScript names). So the commit is the LAST thing +//! the iteration does, after every side exit it can take: +//! +//! * mid-iteration exit ⇒ the real slot still holds the previous commit, which +//! is exactly this iteration's entry value, and the slow clone advances it +//! once; +//! * normal exit ⇒ the last iteration committed, so the real slot holds the +//! final value the code after the loop reads. +//! +//! ## Why the recurrence is evaluated in i64 +//! +//! JavaScript evaluates `(cursor * 17 + 7) % length` in doubles. `frem` on +//! aarch64 is a libm `fmod` call — inside this clone a call is not a slow path, +//! it DELETES the clone (#7690) — so the recurrence is folded to one affine +//! pair `(a, b)` and evaluated as `srem i64`. +//! +//! That is only equal to what JavaScript computes while every intermediate is +//! exactly representable as a double, so the matcher tracks the maximum +//! magnitude any sub-expression can reach over `carried ∈ [0, i32::MAX]` and +//! declines above 2^53. (Fitting i64, which the original design note asked for, +//! is necessary but NOT sufficient: `c * 1e10 + 1` fits i64 comfortably and +//! loses its low bits in the f64 JavaScript actually runs.) `a` and `b` are +//! also required non-negative, so the dividend is non-negative and `srem` +//! agrees with JS `%` — which returns a NEGATIVE remainder for a negative +//! dividend, i.e. an out-of-bounds subscript, not a slow path. + +use anyhow::Result; +use perry_hir::{BinaryOp, Expr, Stmt}; + +use super::loops::{local_bound_is_loop_invariant, local_has_readable_slot}; +use crate::expr::FnCtx; +use crate::types::{DOUBLE, I32, I64}; + +/// The largest integer magnitude an `f64` represents exactly. Above it the i64 +/// recurrence and the f64 one JavaScript runs can disagree. +const EXACT_F64_INTEGER_LIMIT: i64 = 9_007_199_254_740_992; // 2^53 + +/// The preheader materializes the entry value in `0..=i32::MAX`, so this bounds +/// every `carried` the recurrence can ever see. +const MAX_CARRIED: i64 = i32::MAX as i64; + +/// A matched `carried = % m` update. +#[derive(Clone, Copy, Debug)] +pub(super) struct MatchedCarried { + pub carried_id: u32, + pub modulus_id: u32, + pub coeff_a: i64, + pub coeff_b: i64, +} + +/// Fold `expr` to `a * carried + b`, tracking the largest magnitude ANY +/// sub-expression can reach for `carried ∈ [0, i32::MAX]`. +/// +/// Returns `(a, b, max_abs)`. `None` declines — an unknown node, a second +/// variable, a non-integral literal, or any overflow of the checked i64 +/// arithmetic used to compute the bounds. +/// +/// The magnitude is tracked per NODE rather than taken from the folded result +/// because folding can cancel: `c * 1000 - c * 999` is `1 * c`, but JavaScript +/// still evaluates both thousand-fold products. +fn affine_over_carried(expr: &Expr, carried_id: u32) -> Option<(i64, i64, i64)> { + let magnitude = |a: i64, b: i64| -> Option { + a.checked_abs()? + .checked_mul(MAX_CARRIED)? + .checked_add(b.checked_abs()?) + }; + match expr { + Expr::LocalGet(id) if *id == carried_id => Some((1, 0, MAX_CARRIED)), + Expr::Integer(k) => Some((0, *k, k.checked_abs()?)), + Expr::Number(n) if n.is_finite() && n.fract() == 0.0 => { + // `as i64` saturates rather than wrapping, so an out-of-range + // literal would fold to i64::MAX and read as a huge-but-valid + // coefficient. Range-check before converting. + if *n < -(EXACT_F64_INTEGER_LIMIT as f64) || *n > EXACT_F64_INTEGER_LIMIT as f64 { + return None; + } + let k = *n as i64; + Some((0, k, k.checked_abs()?)) + } + Expr::Binary { op, left, right } => { + let (la, lb, lmax) = affine_over_carried(left, carried_id)?; + let (ra, rb, rmax) = affine_over_carried(right, carried_id)?; + let (a, b) = match op { + BinaryOp::Add => (la.checked_add(ra)?, lb.checked_add(rb)?), + BinaryOp::Sub => (la.checked_sub(ra)?, lb.checked_sub(rb)?), + // A product is affine only when one side is a constant; `c * c` + // is not, and neither is `c * m` for a runtime `m`. + BinaryOp::Mul if la == 0 => (ra.checked_mul(lb)?, rb.checked_mul(lb)?), + BinaryOp::Mul if ra == 0 => (la.checked_mul(rb)?, lb.checked_mul(rb)?), + _ => return None, + }; + let here = magnitude(a, b)?; + Some((a, b, lmax.max(rmax).max(here))) + } + _ => None, + } +} + +/// Does `expr` read `id`? +/// +/// Conservative by construction: an expression node this does not enumerate +/// answers "yes". The alternative shape — a catch-all that returns `false` — +/// is the #6377 / `collect_local_refs_expr` failure mode CLAUDE.md names, and +/// here it would let an accumulator read be folded into a statement that no +/// longer sees the value it was written against. +pub(super) fn expr_reads_local(expr: &Expr, id: u32) -> bool { + match expr { + Expr::LocalGet(read) => *read == id, + Expr::Number(_) | Expr::Integer(_) => false, + Expr::Binary { left, right, .. } + | Expr::MathImul(left, right) + | Expr::MathPow(left, right) => expr_reads_local(left, id) || expr_reads_local(right, id), + Expr::NumberCoerce(operand) => expr_reads_local(operand, id), + Expr::MathMin(values) | Expr::MathMax(values) => { + values.iter().any(|e| expr_reads_local(e, id)) + } + Expr::MathAbs(value) + | Expr::MathSqrt(value) + | Expr::MathFloor(value) + | Expr::MathCeil(value) + | Expr::MathRound(value) + | Expr::MathTrunc(value) + | Expr::MathSign(value) + | Expr::MathF16round(value) => expr_reads_local(value, id), + Expr::PropertyGet { object, .. } => expr_reads_local(object, id), + Expr::IndexGet { object, index } => { + expr_reads_local(object, id) || expr_reads_local(index, id) + } + Expr::Conditional { + condition, + then_expr, + else_expr, + } => { + expr_reads_local(condition, id) + || expr_reads_local(then_expr, id) + || expr_reads_local(else_expr, id) + } + _ => true, + } +} + +/// Match the body's first statement as `carried = % m`. +#[allow(clippy::too_many_arguments)] +pub(super) fn match_carried_update( + ctx: &FnCtx<'_>, + stmt: &Stmt, + counter_id: u32, + condition: &Expr, + update: Option<&Expr>, + body: &[Stmt], +) -> Option { + let Stmt::Expr(Expr::LocalSet(carried_id, value)) = stmt else { + return None; + }; + let Expr::Binary { + op: BinaryOp::Mod, + left, + right, + } = value.as_ref() + else { + return None; + }; + let Expr::LocalGet(modulus_id) = right.as_ref() else { + return None; + }; + if *carried_id == counter_id || *modulus_id == counter_id || *modulus_id == *carried_id { + return None; + } + + let (coeff_a, coeff_b, max_abs) = affine_over_carried(left, *carried_id)?; + // A negative dividend makes JS `%` return a negative remainder — an + // out-of-bounds subscript the preheader's `m <= length` says nothing about. + // With `carried >= 0` (the preheader materializes it in `0..=i32::MAX`) and + // both coefficients non-negative, the dividend cannot be negative. + if coeff_a < 0 || coeff_b < 0 { + return None; + } + if max_abs > EXACT_F64_INTEGER_LIMIT { + return None; + } + + // The binding must be a plain, directly-addressable local the clone can + // write back to with one store. A module global would need the rooted-store + // path, a boxed or captured binding lives in a cell a plain store would + // leave stale for an observer outside the clone. + if !ctx.locals.contains_key(carried_id) + || ctx.module_globals.contains_key(carried_id) + || ctx.boxed_vars.contains(carried_id) + || ctx.closure_captures.contains_key(carried_id) + || !local_has_readable_slot(ctx, *carried_id) + || ctx.pod_records.contains_key(carried_id) + || ctx.scalar_replaced.contains_key(carried_id) + { + return None; + } + // `carried` is written exactly once per iteration — by the statement above. + // Every other statement the body matcher admits writes the accumulator, and + // it rejects a body whose accumulator IS the carried local, so the only way + // a second write could appear is a `for` update expression naming it. + if update.is_some_and(|u| expr_writes_local(u, *carried_id)) { + return None; + } + // The modulus is materialized ONCE in the preheader and every iteration's + // `srem` uses that value, so a body (or update, or condition) that could + // rewrite it would derive indices against a stale bound. + if ctx.boxed_vars.contains(modulus_id) + || ctx.closure_captures.contains_key(modulus_id) + || !(local_has_readable_slot(ctx, *modulus_id) + || ctx.module_globals.contains_key(modulus_id)) + || !local_bound_is_loop_invariant(condition, update, body, *modulus_id) + { + return None; + } + + Some(MatchedCarried { + carried_id: *carried_id, + modulus_id: *modulus_id, + coeff_a, + coeff_b, + }) +} + +/// Does `expr` assign to `id`? Conservative in the same direction as +/// [`expr_reads_local`] — an unenumerated node answers "yes". +fn expr_writes_local(expr: &Expr, id: u32) -> bool { + match expr { + Expr::LocalSet(target, _) => *target == id, + Expr::Update { id: target, .. } => *target == id, + Expr::LocalGet(_) | Expr::Number(_) | Expr::Integer(_) => false, + Expr::Binary { left, right, .. } => { + expr_writes_local(left, id) || expr_writes_local(right, id) + } + _ => true, + } +} + +/// Lower the fast clone's carried-index statements, or report that this +/// expression is not one. +/// +/// Two shapes, both synthesized or admitted by the matcher, and neither lowered +/// generically: +/// +/// * the UPDATE `carried = % m` — one `mul`/`add`/`srem` chain in i64 +/// writing the clone's private i32 slot, never the real binding; +/// * the COMMIT `carried = carried`, the trailing statement the matcher appends +/// to the synthesized fast body — one `sitofp` + `store` that publishes the +/// iteration's value to the real slot. +/// +/// The commit is spelled as a self-assignment because it has to be a statement +/// the generic lowering would also accept (the slow clone never sees it — the +/// facts are popped first — but nothing may depend on that). A user-written +/// `c = c` inside a body cannot reach here: the body matcher admits only the +/// carried update, one virtual binding and accumulator writes, and `c` is not +/// the accumulator. +pub(super) fn lower_virtual_carried_stmt(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { + let Expr::LocalSet(id, value) = expr else { + return Ok(false); + }; + let Some((carried, scope_id)) = ctx.element_shape_loop_facts.iter().rev().find_map(|fact| { + let crate::expr::ElementShapeIndex::Carried(carried) = &fact.index else { + return None; + }; + (carried.local_id == *id).then(|| (carried.clone(), fact.scope_id)) + }) else { + return Ok(false); + }; + + if matches!(value.as_ref(), Expr::LocalGet(read) if *read == *id) { + // COMMIT. The value published here is the one every read in this + // iteration already used, so the slot and the binding agree from the + // end of the iteration onward. + let slot = carried.slot.clone(); + let commit_slot = carried.commit_slot.clone(); + let blk = ctx.block(); + let current = blk.load(I32, &slot); + let boxed = blk.sitofp(I32, ¤t, DOUBLE); + blk.store(DOUBLE, &boxed, &commit_slot); + return Ok(true); + } + + // UPDATE. `0 <= carried <= i32::MAX` and `1 <= m <= i32::MAX` hold by the + // preheader's two materializations, and the matcher proved + // `a * i32::MAX + b < 2^53`, so the i64 chain cannot overflow and agrees + // with the f64 arithmetic JavaScript performs. + let slot = carried.slot.clone(); + let modulus_i32 = carried.modulus_i32.clone(); + let coeff_a = carried.coeff_a.to_string(); + let coeff_b = carried.coeff_b.to_string(); + { + let blk = ctx.block(); + let current = blk.load(I32, &slot); + let current64 = blk.sext(I32, ¤t, I64); + let scaled = blk.mul(I64, ¤t64, &coeff_a); + let shifted = blk.add(I64, &scaled, &coeff_b); + let modulus64 = blk.sext(I32, &modulus_i32, I64); + let wrapped = blk.srem(I64, &shifted, &modulus64); + let next = blk.trunc(I64, &wrapped, I32); + blk.store(I32, &next, &slot); + } + // The index for this iteration is now in its slot, so a body with no alias + // binding hangs the shared element prologue off the update itself. + super::element_shape_loop::emit_element_prefetch_for(ctx, scope_id, *id); + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn local(id: u32) -> Expr { + Expr::LocalGet(id) + } + fn mul(l: Expr, r: Expr) -> Expr { + Expr::Binary { + op: BinaryOp::Mul, + left: Box::new(l), + right: Box::new(r), + } + } + fn add(l: Expr, r: Expr) -> Expr { + Expr::Binary { + op: BinaryOp::Add, + left: Box::new(l), + right: Box::new(r), + } + } + fn sub(l: Expr, r: Expr) -> Expr { + Expr::Binary { + op: BinaryOp::Sub, + left: Box::new(l), + right: Box::new(r), + } + } + + #[test] + fn the_benchmark_recurrence_folds() { + // `(cursor * 17 + 7)` + let e = add(mul(local(9), Expr::Integer(17)), Expr::Integer(7)); + assert_eq!( + affine_over_carried(&e, 9), + Some((17, 7, 17 * MAX_CARRIED + 7)) + ); + } + + #[test] + fn a_constant_only_expression_folds_to_b() { + assert_eq!(affine_over_carried(&Expr::Integer(5), 9), Some((0, 5, 5))); + } + + #[test] + fn a_second_variable_declines() { + assert_eq!(affine_over_carried(&add(local(9), local(10)), 9), None); + assert_eq!(affine_over_carried(&mul(local(9), local(9)), 9), None); + } + + /// The magnitude bound is per NODE, not of the folded pair: cancellation + /// must not hide an intermediate JavaScript really computes in f64. + #[test] + fn cancelling_products_still_report_their_intermediates() { + // `c * 1000000000 - c * 999999999` folds to `1 * c`, but both products + // reach ~2e18 for a large `c` — far past 2^53, where the f64 JS runs + // and this i64 chain stop agreeing. + let e = sub( + mul(local(9), Expr::Integer(1_000_000_000)), + mul(local(9), Expr::Integer(999_999_999)), + ); + let (a, b, max_abs) = affine_over_carried(&e, 9).expect("affine"); + assert_eq!((a, b), (1, 0)); + assert!( + max_abs > EXACT_F64_INTEGER_LIMIT, + "the intermediate magnitude must be reported, not the folded one" + ); + } + + #[test] + fn a_non_integral_literal_declines() { + assert_eq!(affine_over_carried(&Expr::Number(1.5), 9), None); + assert_eq!(affine_over_carried(&Expr::Number(f64::NAN), 9), None); + assert_eq!(affine_over_carried(&Expr::Number(f64::INFINITY), 9), None); + // Saturating `as i64` would turn this into a plausible coefficient. + assert_eq!(affine_over_carried(&Expr::Number(1e300), 9), None); + } + + #[test] + fn reads_are_detected_conservatively() { + assert!(expr_reads_local(&add(local(9), Expr::Integer(1)), 9)); + assert!(!expr_reads_local(&add(local(8), Expr::Integer(1)), 9)); + // An unenumerated node answers "reads it". + assert!(expr_reads_local(&Expr::This, 9)); + } +} diff --git a/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs b/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs new file mode 100644 index 0000000000..429523b2b5 --- /dev/null +++ b/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs @@ -0,0 +1,650 @@ +//! #10185: the `fields` and `random` access shapes — IR census for the +//! element-shape clone's loop-carried index, its multi-statement accumulator +//! fold, and its two non-numeric reads. +//! +//! A child module of `element_shape_shape_keyed_tests` (see the `mod` +//! declaration there), so `use super::*` brings both that file's shape-keyed +//! assertions and `element_shape_loop_tests`'s slicing helpers. +//! +//! Every positive here is paired with a sabotage case, and the two liveness +//! assertions #10171 was short — the preheader's EXACT comparison set, and that +//! the residual check really is shared rather than repeated — are the ones that +//! would have caught a clone that is emitted, branched into, and never entered +//! or never actually cheaper. + +use super::*; + +const ROWS2_ID: u32 = 40; +const COUNT2_ID: u32 = 41; +const MOD2_ID: u32 = 42; +const SUM2_ID: u32 = 43; +const COUNTER2_ID: u32 = 44; +const CURSOR_ID: u32 = 45; +const ALIAS_ID: u32 = 46; +const OTHER_ID: u32 = 47; + +/// `rows[].` +fn rows_field(index: Expr, prop: &str) -> Expr { + Expr::PropertyGet { + object: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ROWS2_ID)), + index: Box::new(index), + }), + property: prop.to_string(), + byte_offset: 0, + } +} + +/// `sum = sum + ` +fn add_to_sum(value: Expr) -> Stmt { + Stmt::Expr(Expr::LocalSet( + SUM2_ID, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(SUM2_ID)), + right: Box::new(value), + }), + )) +} + +fn binary(op: BinaryOp, left: Expr, right: Expr) -> Expr { + Expr::Binary { + op, + left: Box::new(left), + right: Box::new(right), + } +} + +/// `cursor = % n` +fn cursor_update(affine: Expr) -> Stmt { + Stmt::Expr(Expr::LocalSet( + CURSOR_ID, + Box::new(binary(BinaryOp::Mod, affine, Expr::LocalGet(MOD2_ID))), + )) +} + +/// The benchmark's own recurrence, `(cursor * 17 + 7)`. +fn benchmark_affine() -> Expr { + binary( + BinaryOp::Add, + binary(BinaryOp::Mul, Expr::LocalGet(CURSOR_ID), Expr::Integer(17)), + Expr::Integer(7), + ) +} + +/// `const index = cursor;` +fn cursor_alias() -> Stmt { + Stmt::Let { + id: ALIAS_ID, + name: "index".to_string(), + ty: Type::Number, + mutable: false, + init: Some(Expr::LocalGet(CURSOR_ID)), + } +} + +/// `const index = i % n;` +fn modulo_index() -> Stmt { + Stmt::Let { + id: ALIAS_ID, + name: "index".to_string(), + ty: Type::Any, + mutable: false, + init: Some(binary( + BinaryOp::Mod, + Expr::LocalGet(COUNTER2_ID), + Expr::LocalGet(MOD2_ID), + )), + } +} + +/// ```text +/// function run(rows: any, count: number, n: number): number { +/// let sum = 0; +/// let cursor = 0; +/// for (let i = 0; i < count; i++) +/// return sum; +/// } +/// ``` +fn access_module(body: Vec) -> Module { + let mut m = Module::new("element_shape_loop.ts"); + let param = |id: u32, name: &str, ty: Type| perry_hir::Param { + id, + name: name.to_string(), + ty, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }; + m.functions = vec![perry_hir::Function { + id: 902, + name: "run".to_string(), + type_params: Vec::new(), + params: vec![ + param(ROWS2_ID, "rows", Type::Any), + param(COUNT2_ID, "count", Type::Number), + param(MOD2_ID, "n", Type::Number), + ], + return_type: Type::Number, + body: vec![ + Stmt::Let { + id: SUM2_ID, + name: "sum".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + }, + Stmt::Let { + id: CURSOR_ID, + name: "cursor".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + }, + Stmt::For { + init: Some(Box::new(Stmt::Let { + id: COUNTER2_ID, + name: "i".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + })), + condition: Some(Expr::Compare { + op: CompareOp::Lt, + left: Box::new(Expr::LocalGet(COUNTER2_ID)), + right: Box::new(Expr::LocalGet(COUNT2_ID)), + }), + update: Some(Expr::Update { + id: COUNTER2_ID, + op: UpdateOp::Increment, + prefix: false, + }), + body, + }, + Stmt::Return(Some(Expr::LocalGet(SUM2_ID))), + ], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }]; + m.init_kind = ModuleInitKind::Eager; + m +} + +/// The benchmark's `random` body, with and without the alias spelling. +fn random_body(with_alias: bool) -> Vec { + let index = if with_alias { ALIAS_ID } else { CURSOR_ID }; + let mut body = vec![cursor_update(benchmark_affine())]; + if with_alias { + body.push(cursor_alias()); + } + body.push(add_to_sum(rows_field(Expr::LocalGet(index), "id"))); + body +} + +/// The benchmark's `fields` body: three accumulator statements over ONE +/// element, one through a string and one through a boolean. +fn fields_body() -> Vec { + vec![ + modulo_index(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + add_to_sum(Expr::PropertyGet { + object: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "name")), + property: "length".to_string(), + byte_offset: 0, + }), + add_to_sum(Expr::Conditional { + condition: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "active")), + then_expr: Box::new(Expr::Integer(1)), + else_expr: Box::new(Expr::Integer(0)), + }), + ] +} + +/// How many side exits the clone can take per iteration — one branch per +/// residual / tag test that leaves for the slow preheader. +fn side_exit_count(fast: &str) -> usize { + fast.matches("label %element_shape.loop.slow.preheader") + .count() +} + +/// The LAST block in `fast` whose label starts with `prefix`, body only. +/// +/// Used to name the block a commit must live in. Reaching a block named after a +/// passed tag test IS the proof that every side exit of the iteration is behind +/// it — which is stronger than any textual before/after comparison, because the +/// emitted block ORDER is not the execution order (the loop's `update` block is +/// printed before the body blocks that branch into it). +fn last_block_with_prefix<'a>(fast: &'a str, prefix: &str) -> &'a str { + let start = fast + .rmatch_indices('\n') + .map(|(idx, _)| idx + 1) + .find(|&idx| { + fast[idx..].starts_with(prefix) + && fast[idx..] + .lines() + .next() + .unwrap_or("") + .trim_end() + .ends_with(':') + }) + .unwrap_or_else(|| panic!("no block named `{prefix}*` in:\n{fast}")); + let body = &fast[start..]; + let end = body.find("\n\n").unwrap_or(body.len()); + &body[..end] +} + +/// Count the `store` instructions one iteration of the clone executes. +fn store_count(fast: &str) -> usize { + fast.lines() + .filter(|l| l.trim_start().starts_with("store ")) + .count() +} + +// --------------------------------------------------------------------------- +// #10185 — the LOOP-CARRIED index (`random`). +// --------------------------------------------------------------------------- + +#[test] +fn a_carried_recurrence_gets_a_shape_keyed_clone() { + let ir = emit(&access_module(random_body(true))); + assert_shape_keyed_clone(&ir, "carried index with an alias"); + let fast = fast_clone_slice(&ir); + assert!( + fast.contains("srem i64") && fast.contains("mul i64"), + "the recurrence must be the i64 chain, not an `frem` (a libm call on \ + aarch64, which would delete the clone); emitted:\n{fast}" + ); + assert!( + !fast.contains("frem"), + "an `frem` in the clone is the whole reason this form needed its own \ + lowering; emitted:\n{fast}" + ); + assert!( + ir.contains("element_shape.loop.carried.range"), + "the carried ENTRY value must be materialized as a validated \ + non-negative i32 — the i64 bound and the `srem` sign argument are \ + both stated over `0..=i32::MAX`" + ); + assert!( + ir.contains("element_shape.loop.modulus.range"), + "the modulus must be materialized: `srem` by zero is UB and `x % 0` \ + is NaN in JS" + ); +} + +#[test] +fn a_carried_recurrence_matches_without_the_alias() { + // `rows[cursor]` directly — the alias is sugar, not part of the form. + let ir = emit(&access_module(random_body(false))); + assert_shape_keyed_clone(&ir, "carried index, no alias"); + let fast = fast_clone_slice(&ir); + assert!(fast.contains("srem i64"), "emitted:\n{fast}"); +} + +/// THE correctness assertion for this form. +/// +/// The residual side exit resumes the CURRENT iteration in the slow clone, +/// which re-runs the whole body — the recurrence included. So the write-back to +/// the real binding must come after EVERY exit the iteration can take, or the +/// recurrence is applied twice and every later index is silently wrong (still +/// in bounds, still a valid record, just not the one JavaScript names). +#[test] +fn the_carried_write_back_follows_every_side_exit() { + let ir = emit(&access_module(random_body(true))); + let fast = fast_clone_slice(&ir); + // Every side exit this iteration can take is a tag test on a loaded word, + // and the LAST of them is the `id` read's Number test. Its admitted arm — + // `element_shape.number.*` — is therefore reachable only once the whole + // iteration is past every exit, so a commit inside it is provably last. + let committed = last_block_with_prefix(&fast, "element_shape.number"); + assert!( + committed.contains("sitofp i32") && committed.contains("store double"), + "the carried write-back must live past every side exit; a write-back \ + before one makes the slow clone apply the recurrence twice for that \ + iteration, and every later index is silently wrong. \ + final block:\n{committed}\nfull clone:\n{fast}" + ); + assert_eq!( + committed.matches("store double").count(), + 1, + "the carried value must be published exactly once per iteration; \ + final block:\n{committed}" + ); + assert_eq!( + store_count(&fast), + // the carried i32 slot, the prefetched element handle, the + // accumulator, the carried write-back, and the counter's increment + 5, + "a second write to the carried binding would double-apply the \ + recurrence on a side exit; emitted:\n{fast}" + ); +} + +/// The same liveness assertion #10123's derived index carries: the ONE length +/// comparison the preheader owes is `modulus <= length`, NOT the counter arm's +/// `length >= bound`. Demanding the latter makes the clone unenterable whenever +/// the loop runs more times than the array is long — which is every access +/// benchmark. +#[test] +fn a_carried_index_owes_exactly_the_modulus_obligation() { + let ir = emit(&access_module(random_body(true))); + let deref = block_slice(&ir, "element_shape.loop.preheader.deref"); + assert_eq!( + deref.matches("icmp uge i32").count(), + 1, + "a carried index owes exactly ONE length comparison (`modulus <= \ + length`); emitted:\n{deref}" + ); + let fast = fast_clone_slice(&ir); + assert!( + !fast.contains("icmp ult i32") && !fast.contains("icmp ugt i32"), + "the bounds obligation is discharged ONCE in the preheader; a per-read \ + test in the clone would mean it was not; emitted:\n{fast}" + ); +} + +#[test] +fn a_negative_coefficient_declines() { + // `(cursor * 17 - 7) % n` can produce a negative dividend, and JS `%` + // returns a NEGATIVE remainder for one — an out-of-bounds subscript the + // preheader's `m <= length` says nothing about. + let body = vec![ + cursor_update(binary( + BinaryOp::Sub, + binary(BinaryOp::Mul, Expr::LocalGet(CURSOR_ID), Expr::Integer(17)), + Expr::Integer(7), + )), + cursor_alias(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a recurrence whose dividend can go negative must decline" + ); +} + +#[test] +fn a_non_affine_recurrence_declines() { + // `(cursor * cursor) % n` is not affine, so no (a, b) pair describes it and + // no magnitude bound can be proven. + let body = vec![ + cursor_update(binary( + BinaryOp::Mul, + Expr::LocalGet(CURSOR_ID), + Expr::LocalGet(CURSOR_ID), + )), + cursor_alias(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a non-affine recurrence must decline" + ); +} + +#[test] +fn a_recurrence_that_leaves_exact_double_range_declines() { + // `cursor * 1e9 + 1` fits i64 comfortably and does NOT fit the f64 that + // JavaScript actually evaluates it in, so the i64 chain and the program + // would disagree. "Fits i64" was the original design note's obligation; it + // is necessary and not sufficient. + let body = vec![ + cursor_update(binary( + BinaryOp::Add, + binary( + BinaryOp::Mul, + Expr::LocalGet(CURSOR_ID), + Expr::Integer(1_000_000_000), + ), + Expr::Integer(1), + )), + cursor_alias(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a recurrence whose intermediates pass 2^53 must decline: JS evaluates \ + it in doubles and the clone's i64 chain would not agree" + ); +} + +#[test] +fn a_bare_read_of_the_carried_local_declines() { + // Inside the clone the real slot is one iteration behind until the trailing + // commit, so a bare `cursor` in the accumulator would read a stale value. + let body = vec![ + cursor_update(benchmark_affine()), + cursor_alias(), + add_to_sum(binary( + BinaryOp::Add, + rows_field(Expr::LocalGet(ALIAS_ID), "id"), + Expr::LocalGet(CURSOR_ID), + )), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a bare read of the carried local must decline the clone" + ); +} + +#[test] +fn a_recurrence_over_a_second_variable_declines() { + // `(cursor + other) % n` is not a function of `cursor` alone; `other` is + // not materialized, bounded, or proven loop-invariant. + let body = vec![ + cursor_update(binary( + BinaryOp::Add, + Expr::LocalGet(CURSOR_ID), + Expr::LocalGet(OTHER_ID), + )), + cursor_alias(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "a recurrence reading a second variable must decline" + ); +} + +// --------------------------------------------------------------------------- +// #10185 — the MULTI-STATEMENT body, the string `.length` and the ternary +// (`fields`). +// --------------------------------------------------------------------------- + +#[test] +fn the_fields_body_gets_a_shape_keyed_clone() { + let ir = emit(&access_module(fields_body())); + assert_shape_keyed_clone(&ir, "three reads of one element"); + let fast = fast_clone_slice(&ir); + assert!( + fast.contains("element_shape.strlen.heap") && fast.contains("element_shape.strlen.sso"), + "a string `.length` must decode BOTH representations inline — a call \ + to the runtime would delete the clone; emitted:\n{fast}" + ); + assert!( + fast.contains("select i1"), + "the ternary must become a select between the two constants; \ + emitted:\n{fast}" + ); + assert!( + fast.contains("9222246136947933188") && fast.contains("9222246136947933187"), + "the ternary must test the two boolean singletons by exact NaN-box bit \ + pattern; guessing JS truthiness in the clone would be a miscompile; \ + emitted:\n{fast}" + ); +} + +/// THE cost assertion. Three reads of `rows[index]` are three loads of the same +/// element and three repeats of the residual header/ShapeId check unless the +/// prologue shares them — and a clone that is entered but pays 3x the guard is +/// exactly the "emitted, branched into, and not actually cheaper" failure the +/// IR census exists to catch. +#[test] +fn three_reads_of_one_element_share_one_residual_check() { + let ir = emit(&access_module(fields_body())); + let fast = fast_clone_slice(&ir); + assert_eq!( + fast.matches("134250751").count(), + 1, + "the shape-keyed residual mask must appear ONCE per iteration, not \ + once per read; emitted:\n{fast}" + ); + assert_eq!( + fast.matches("element_shape.load").count(), + // one block definition plus the `br` that targets it + 2, + "there must be exactly one residual-check block in the iteration; \ + emitted:\n{fast}" + ); + // Four tag/residual tests, not twelve: the residual once, then one + // representation test per read. + assert_eq!( + side_exit_count(&fast), + 4, + "one residual check plus one tag test per read; emitted:\n{fast}" + ); +} + +/// THE correctness assertion for the multi-statement body. `sum += a; sum += b` +/// commits `sum` twice; if the second read's tag test side-exits, the slow +/// clone re-runs the iteration and applies `a` a second time. The fold makes +/// the whole iteration commit once, after every exit. +#[test] +fn the_accumulator_commits_once_after_every_side_exit() { + let ir = emit(&access_module(fields_body())); + let fast = fast_clone_slice(&ir); + assert_eq!( + fast.matches("fadd double").count(), + 3, + "the fold keeps all three additions in source order — it changes when \ + the result is STORED, never what is computed; emitted:\n{fast}" + ); + assert_eq!( + store_count(&fast), + // the derived index, the prefetched element handle, the accumulator, + // and the counter's own increment + 4, + "three accumulator statements must fold to ONE accumulator store; a \ + second one means a side exit from a later read can leave an earlier \ + read already applied when the slow clone re-runs the iteration; \ + emitted:\n{fast}" + ); + // The one store lives in the block the LAST tag test branches into, so it + // is behind every side exit of the iteration. + let committed = last_block_with_prefix(&fast, "element_shape.bool"); + assert!( + committed.contains("store "), + "the accumulator store must live past every side exit; final \ + block:\n{committed}\nfull clone:\n{fast}" + ); +} + +#[test] +fn an_addend_that_reads_the_accumulator_declines() { + // `sum += rows[i].id; sum += sum` — folding would substitute the PRE-first + // -statement value of `sum` into the second addend. + let body = vec![ + modulo_index(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + add_to_sum(Expr::LocalGet(SUM2_ID)), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "an addend reading the accumulator must decline the fold, and with it \ + the clone" + ); +} + +#[test] +fn two_accumulators_decline() { + let body = vec![ + modulo_index(), + add_to_sum(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + Stmt::Expr(Expr::LocalSet( + CURSOR_ID, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(CURSOR_ID)), + right: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + }), + )), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "the body must write exactly one accumulator" + ); +} + +#[test] +fn a_ternary_with_a_non_constant_arm_declines() { + let body = vec![ + modulo_index(), + add_to_sum(Expr::Conditional { + condition: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "active")), + then_expr: Box::new(rows_field(Expr::LocalGet(ALIAS_ID), "id")), + else_expr: Box::new(Expr::Integer(0)), + }), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "only integer-constant ternary arms are admitted" + ); +} + +#[test] +fn a_ternary_on_something_other_than_an_element_read_declines() { + let body = vec![ + modulo_index(), + add_to_sum(Expr::Conditional { + condition: Box::new(Expr::LocalGet(COUNTER2_ID)), + then_expr: Box::new(Expr::Integer(1)), + else_expr: Box::new(Expr::Integer(0)), + }), + ]; + let ir = emit(&access_module(body)); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "the clone lowers a boolean-singleton test, not general JS truthiness" + ); +} + +/// The two non-numeric reads are SHAPE-keyed only: a class-keyed clone's slot +/// holds a raw `double`, so there is no NaN-box tag to test and both readers +/// would be decoding a double's bit pattern. +#[test] +fn a_class_typed_array_declines_the_string_length_read() { + let ir = emit(&element_shape_module( + vec![Stmt::Expr(Expr::LocalSet( + SUM_ID, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(SUM_ID)), + right: Box::new(Expr::PropertyGet { + object: Box::new(elem_field(ARRAY_ID, Expr::LocalGet(COUNTER_ID))), + property: "length".to_string(), + byte_offset: 0, + }), + }), + ))], + None, + )); + assert!( + !ir.contains("element_shape.strlen"), + "the class-keyed arm reads raw doubles; there is no tag to test" + ); +} diff --git a/crates/perry-codegen/src/stmt/element_shape_loop.rs b/crates/perry-codegen/src/stmt/element_shape_loop.rs index 1ce4f0cd8d..47e54c7c46 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop.rs @@ -152,7 +152,7 @@ use super::loops::{ CLASS_FIELD_LOOP_PROP_DENYLIST, }; use crate::expr::{lower_expr, FnCtx}; -use crate::types::{DOUBLE, I1, I32}; +use crate::types::{DOUBLE, I1, I32, I64}; /// Loop bound: a literal, a loop-invariant local / module global that is /// materialized to i32 once in the preheader, or the tracked array's own @@ -210,14 +210,28 @@ enum ElementShapeIdentity { enum MatchedIndex { Counter, Constant(i64), - DerivedMod { local_id: u32, modulus_id: u32 }, + DerivedMod { + local_id: u32, + modulus_id: u32, + }, + /// #10185: `c = (a*c + b) % m; … arr[c]`, optionally spelled through a + /// `const index = c` alias. The recurrence itself lives on + /// [`ElementShapeVersionedLoop::carried`] — this arm records only the + /// spellings the subscript may take. See `stmt/element_shape_carried.rs`. + Carried { + carried_id: u32, + alias_id: Option, + }, } impl MatchedIndex { /// Mirror of [`crate::expr::ElementShapeIndex::needs_counter_i32_slot`], /// asked before the fact exists. fn needs_counter_i32_slot(self) -> bool { - !matches!(self, MatchedIndex::Constant(_)) + !matches!( + self, + MatchedIndex::Constant(_) | MatchedIndex::Carried { .. } + ) } } @@ -235,6 +249,21 @@ struct ElementShapeVersionedLoop { /// form; `None` for the original single-statement accumulator body. element_binding: Option, accumulator_id: u32, + /// #10185: the matched recurrence, for the [`MatchedIndex::Carried`] form. + carried: Option, + /// #10185: the body the FAST clone lowers, when it is not the source body. + /// + /// Two rewrites, both of which exist to put every side exit an iteration + /// can take BEFORE any of its stores: + /// + /// * K accumulator statements fold into one, so a tag test that fails on + /// the third read cannot leave the first two already applied to `acc` + /// when the slow clone re-runs the iteration; + /// * a carried recurrence gains a trailing commit statement, so the real + /// binding is published once the iteration is past every exit. + /// + /// The SLOW clone always lowers the original `body`. + fast_body: Option>, } /// The locals the pure-expression walk reasons about. @@ -246,6 +275,11 @@ struct PureExprScope { /// #10123: `(d, m)` for a body whose first statement is /// `const d = counter % m`. derived: Option<(u32, u32)>, + /// #10185: `(carried, alias)` for a body whose first statement advances a + /// loop-carried index. Both are excluded from bare reads for the same + /// reason `derived` is — inside the clone the real slot is one iteration + /// behind until the trailing commit. + carried: Option<(u32, Option)>, } /// What the walk collects. @@ -311,8 +345,33 @@ fn element_shape_loop_pure_expr_collect( out.props.insert(property.clone()); true } + // #10185: `arr[].prop.length` — a STRING field's JS length. + // Admitted as its own form rather than as "a read of `prop` + // followed by a generic `.length`", because the generic one is a + // property diamond ending in a runtime call, and a call inside this + // clone deletes it. The emitted read tag-tests the loaded word for + // both string representations and side-exits otherwise + // (`expr::element_shape_reads`). + Expr::PropertyGet { .. } if property == "length" => { + element_shape_loop_pure_expr_collect(ctx, object, scope, out) + } _ => false, }, + // #10185: `arr[].prop ? A : B` with constant arms. JS truthiness + // of an arbitrary value is a runtime question the clone is not allowed + // to guess, so the emitted read admits ONLY the two boolean singletons + // and side-exits on everything else. Anything but a tracked element + // read in the condition, or a non-constant arm, declines. + Expr::Conditional { + condition, + then_expr, + else_expr, + } => { + matches!(condition.as_ref(), Expr::PropertyGet { .. }) + && crate::expr::element_shape_reads::integer_arm(then_expr).is_some() + && crate::expr::element_shape_reads::integer_arm(else_expr).is_some() + && element_shape_loop_pure_expr_collect(ctx, condition, scope, out) + } // A bare read of the array, the counter, the element binding or the // derived index as a VALUE could flow it into arbitrary lowering; only // scalar reads the analysis proves numeric are admitted. The element @@ -324,6 +383,10 @@ fn element_shape_loop_pure_expr_collect( Expr::LocalGet(id) => { scope.element_binding != Some(*id) && scope.derived.map(|(d, _)| d) != Some(*id) + // #10185: the carried index and its alias are virtual too — + // their real slots are one iteration behind inside the clone, + // so a bare read would hand out a stale value. + && scope.carried.is_none_or(|(c, alias)| *id != c && alias != Some(*id)) && out.array.is_none_or(|a| a != *id) && (*id == scope.accumulator_id || crate::type_analysis::is_numeric_expr(ctx, expr)) } @@ -381,7 +444,18 @@ fn match_index_form(index: &perry_hir::Expr, scope: &PureExprScope) -> Option None, + // #10185: `arr[c]` and `const index = c; arr[index]` are the same + // subscript; the alias is virtual and carries no obligation of its + // own. + _ => match scope.carried { + Some((carried_id, alias_id)) if carried_id == *id || alias_id == Some(*id) => { + Some(MatchedIndex::Carried { + carried_id, + alias_id, + }) + } + _ => None, + }, }, // `0..=i32::MAX`, so the preheader's `length > k` compare is an i32 // one and the emitted index needs no conversion. @@ -619,41 +693,100 @@ fn anon_shape_field_type_is_compatible( /// Answering `false` for want of the counter's i32 slot cannot happen — the /// matcher requires one for the derived-index form — and would only cost the /// clone, never correctness. -pub(super) fn lower_virtual_clone_binding(ctx: &mut FnCtx<'_>, id: u32) -> bool { - if ctx - .element_shape_loop_facts - .iter() - .any(|fact| fact.element_binding == Some(id)) - { - return true; +pub(super) fn lower_virtual_clone_binding(ctx: &mut FnCtx<'_>, id: u32) -> Result { + enum VirtualBinding { + /// #7771 `const r = arr[j]` and #10185 `const index = c`: pure aliases, + /// no instruction of their own. + Alias, + /// #10123 `const d = j % m`: one `srem i32`. + DerivedMod { + counter_id: u32, + modulus_i32: String, + slot: String, + }, } - let Some((counter_slot, modulus_i32, derived_slot)) = - ctx.element_shape_loop_facts.iter().rev().find_map(|fact| { - let crate::expr::ElementShapeIndex::DerivedMod { + + let Some((scope_id, binding)) = ctx.element_shape_loop_facts.iter().rev().find_map(|fact| { + if fact.element_binding == Some(id) { + return Some((fact.scope_id, VirtualBinding::Alias)); + } + match &fact.index { + crate::expr::ElementShapeIndex::DerivedMod { local_id, modulus_i32, slot, - } = &fact.index - else { - return None; - }; - if *local_id != id { - return None; + } if *local_id == id => Some(( + fact.scope_id, + VirtualBinding::DerivedMod { + counter_id: fact.index_local_id, + modulus_i32: modulus_i32.clone(), + slot: slot.clone(), + }, + )), + crate::expr::ElementShapeIndex::Carried(carried) if carried.alias_id == Some(id) => { + Some((fact.scope_id, VirtualBinding::Alias)) } - Some(( - ctx.i32_counter_slots.get(&fact.index_local_id)?.clone(), - modulus_i32.clone(), - slot.clone(), - )) - }) + _ => None, + } + }) else { + return Ok(false); + }; + + match binding { + VirtualBinding::Alias => {} + VirtualBinding::DerivedMod { + counter_id, + modulus_i32, + slot, + } => { + let Some(counter_slot) = ctx.i32_counter_slots.get(&counter_id).cloned() else { + return Ok(false); + }; + let blk = ctx.block(); + let counter = blk.load(I32, &counter_slot); + let derived = blk.srem(I32, &counter, &modulus_i32); + blk.store(I32, &derived, &slot); + } + } + // #10185: this iteration's index is now in its slot, so this is where the + // shared element deref belongs — for the ONE binding that owns it. + emit_element_prefetch_for(ctx, scope_id, id); + Ok(true) +} + +/// #10185: emit the once-per-iteration element prologue, if `site_local` is the +/// statement the fact designated to own it. +/// +/// Every read in the clone loads the handle this parks, so the designated +/// statement MUST run before any of them — which is why the matcher only ever +/// designates the body's leading virtual statement, and why a fact carrying a +/// prefetch whose index cannot be materialized is a compiler bug rather than a +/// missed optimization (the reads would load an uninitialized alloca). The +/// matcher checks `needs_counter_i32_slot` before the fact is built, so the +/// index is always available here. +pub(super) fn emit_element_prefetch_for(ctx: &mut FnCtx<'_>, scope_id: u32, site_local: u32) { + let Some(fact) = ctx + .element_shape_loop_facts + .iter() + .find(|fact| fact.scope_id == scope_id) + .cloned() else { - return false; + return; + }; + let Some(prefetch) = fact.elem_prefetch.clone() else { + return; }; - let blk = ctx.block(); - let counter = blk.load(I32, &counter_slot); - let derived = blk.srem(I32, &counter, &modulus_i32); - blk.store(I32, &derived, &derived_slot); - true + if prefetch.site_local_id != site_local { + return; + } + let idx_i32 = crate::expr::element_shape_guard::emit_element_shape_index(ctx, &fact) + .expect("a prefetching fact always has a materializable index"); + crate::expr::element_shape_guard::emit_element_shape_prefetch( + ctx, + &fact, + &idx_i32, + &prefetch.handle_slot, + ); } /// Is `array_id` an untyped receiver — the shape-keyed arm's entry condition? @@ -799,7 +932,7 @@ fn match_element_shape_versioned_loop( return None; } - // Store-free body, in one of three admitted shapes (see the module docs): + // Store-free body, in one of four admitted shapes (see the module docs): // // 1. `acc = ].field>` — the original // single statement; @@ -807,89 +940,156 @@ fn match_element_shape_versioned_loop( // element-binding form, the shape real read loops are written in; // 3. `const d = j % m; acc = ` — // #10123's derived-index form, the shape every sequential pass over a - // parsed record array is written in. + // parsed record array is written in; + // 4. #10185's loop-carried form, `c = (a*c + b) % m;` optionally followed + // by `const index = c;`. // - // In 2 and 3 the binding is VIRTUAL inside the fast clone: its `Let` - // emits nothing (form 2) or one `srem i32` (form 3) rather than the - // generic lowering (`stmt/let_stmt.rs`), so the revocation argument (no - // store, no call in the clone) is unchanged. `const`-only, deliberately: a - // `var` binding is function-scoped and observable after the loop, where - // the skipped `Let` would leave the slot holding its pre-loop value. + // In 2, 3 and 4 the leading statement is VIRTUAL inside the fast clone: it + // emits nothing (form 2), one `srem i32` (form 3) or the i64 recurrence + // (form 4) rather than the generic lowering (`stmt/let_stmt.rs`, + // `stmt/element_shape_carried.rs`), so the revocation argument (no store, + // no call in the clone) is unchanged. The bindings are `const`-only, + // deliberately: a `var` binding is function-scoped and observable after the + // loop, where the skipped `Let` would leave the slot holding its pre-loop + // value. Form 4's `c` is the one mutable exception, and it pays for it with + // a write-back (see `element_shape_carried`). // + // #10185 also admits K >= 1 accumulator statements instead of exactly one, + // which is what `sum += rows[i].id; sum += rows[i].name.length; …` needs. // NOTHING else is admitted. let mut element_binding: Option<(u32, u32)> = None; let mut derived: Option<(u32, u32)> = None; - let (acc_id, value) = match body { - [Stmt::Expr(Expr::LocalSet(acc_id, value))] => (acc_id, value), - [Stmt::Let { - id, - mutable: false, - init: Some(binding_init), - .. - }, Stmt::Expr(Expr::LocalSet(acc_id, value))] => { - // The binding must be a plain, loop-owned const local in either - // form. A boxed or captured binding lives in a cell the clone's - // replacement `Let` would leave stale for an observer outside the - // clone; a module-global id is not a body-scoped binding at all. - if *id == counter_id - || ctx.boxed_vars.contains(id) - || ctx.module_globals.contains_key(id) - || ctx.closure_captures.contains_key(id) - { - return None; + let mut carried: Option = None; + let mut carried_alias: Option = None; + let mut rest: &[Stmt] = body; + + // (4) The recurrence, which must be the body's FIRST statement: the index + // it produces is read by every statement after it, and a read BEFORE it + // would see a value the preheader bounded but the commit protocol does not + // describe. + if let Some(first) = rest.first() { + if let Some(matched) = super::element_shape_carried::match_carried_update( + ctx, first, counter_id, condition?, update, body, + ) { + carried = Some(matched); + rest = &rest[1..]; + } + } + + // (2)/(3)/(4-alias) The leading virtual binding. + if let Some(Stmt::Let { + id, + mutable: false, + init: Some(binding_init), + .. + }) = rest.first() + { + // The binding must be a plain, loop-owned const local in every form. A + // boxed or captured binding lives in a cell the clone's replacement + // `Let` would leave stale for an observer outside the clone; a + // module-global id is not a body-scoped binding at all. + if *id == counter_id + || ctx.boxed_vars.contains(id) + || ctx.module_globals.contains_key(id) + || ctx.closure_captures.contains_key(id) + { + return None; + } + match binding_init { + Expr::IndexGet { object, index } if carried.is_none() => { + let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = + (object.as_ref(), index.as_ref()) + else { + return None; + }; + // Same receiver/index discipline as the walk's IndexGet + // arm: the fetch must be `arr[counter]` exactly. + if *idx_id != counter_id || *arr_id == counter_id || *id == *arr_id { + return None; + } + element_binding = Some((*id, *arr_id)); } - match binding_init { - Expr::IndexGet { object, index } => { - let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = - (object.as_ref(), index.as_ref()) - else { - return None; - }; - // Same receiver/index discipline as the walk's IndexGet - // arm: the fetch must be `arr[counter]` exactly. - if *idx_id != counter_id || *arr_id == counter_id || *id == *arr_id { - return None; - } - element_binding = Some((*id, *arr_id)); + // #10123. `%` on two locals, the left one the counter. The + // modulus is validated below (it must be readable and + // loop-invariant), and the RANGE obligation — `1 <= m` so the + // `srem` cannot divide by zero, `m <= length` so every derived + // index is in bounds — is discharged in the preheader, which + // is also where a non-number / fractional `m` sends the loop + // to the slow clone. + Expr::Binary { + op: BinaryOp::Mod, + left, + right, + } if carried.is_none() => { + let (Expr::LocalGet(num_id), Expr::LocalGet(modulus_id)) = + (left.as_ref(), right.as_ref()) + else { + return None; + }; + if *num_id != counter_id || *modulus_id == counter_id || *id == *modulus_id { + return None; } - // #10123. `%` on two locals, the left one the counter. The - // modulus is validated below (it must be readable and - // loop-invariant), and the RANGE obligation — `1 <= m` so the - // `srem` cannot divide by zero, `m <= length` so every derived - // index is in bounds — is discharged in the preheader, which - // is also where a non-number / fractional `m` sends the loop - // to the slow clone. - Expr::Binary { - op: BinaryOp::Mod, - left, - right, - } => { - let (Expr::LocalGet(num_id), Expr::LocalGet(modulus_id)) = - (left.as_ref(), right.as_ref()) - else { - return None; - }; - if *num_id != counter_id || *modulus_id == counter_id || *id == *modulus_id { - return None; - } - if !modulus_local_is_admissible(ctx, *modulus_id, condition?, update, body) { - return None; - } - derived = Some((*id, *modulus_id)); + if !modulus_local_is_admissible(ctx, *modulus_id, condition?, update, body) { + return None; } - _ => return None, + derived = Some((*id, *modulus_id)); } - (acc_id, value) + // #10185: `const index = c` after the recurrence. A pure alias, so + // it emits nothing at all — the subscript resolves to the carried + // slot either way. + Expr::LocalGet(aliased) => { + let matched = carried.as_ref()?; + if *aliased != matched.carried_id || *id == matched.modulus_id { + return None; + } + carried_alias = Some(*id); + } + _ => return None, } - _ => return None, - }; + rest = &rest[1..]; + } + + // (1) One or more accumulator statements over the SAME local. + let mut acc_id: Option<&u32> = None; + let mut values: Vec<&perry_hir::Expr> = Vec::new(); + for stmt in rest { + let Stmt::Expr(Expr::LocalSet(id, value)) = stmt else { + return None; + }; + match acc_id { + None => acc_id = Some(id), + Some(seen) if seen == id => {} + Some(_) => return None, + } + values.push(value.as_ref()); + } + let acc_id = acc_id?; if *acc_id == counter_id || !ctx.locals.contains_key(acc_id) || ctx.boxed_vars.contains(acc_id) || ctx.module_globals.contains_key(acc_id) + || carried.is_some_and(|c| c.carried_id == *acc_id || c.modulus_id == *acc_id) { return None; } + // #10185: statements 2..K are folded into statement 1 for the fast clone + // (`acc = a; acc = acc + b` ≡ `acc = (a) + b`), so that the WHOLE iteration + // commits once, after every side exit it can take. Each of them must + // therefore be `acc ` with `acc` as the left operand — the shape + // every compound assignment lowers to — and the right operand must not read + // `acc`, or substituting it would make it see the pre-statement value. + for value in &values[1..] { + let Expr::Binary { left, right, .. } = value else { + return None; + }; + if !matches!(left.as_ref(), Expr::LocalGet(id) if id == acc_id) { + return None; + } + if super::element_shape_carried::expr_reads_local(right, *acc_id) { + return None; + } + } + // The binding form pins the array before the walk runs, so a body mixing // `r.field` with `other[j].field` is declined by the walk's one-array rule. let scope = PureExprScope { @@ -897,15 +1097,22 @@ fn match_element_shape_versioned_loop( accumulator_id: *acc_id, element_binding: element_binding.map(|(id, _)| id), derived, + carried: carried.map(|c| (c.carried_id, carried_alias)), }; - if scope.element_binding == Some(*acc_id) || derived.map(|(d, _)| d) == Some(*acc_id) { + if scope.element_binding == Some(*acc_id) + || derived.map(|(d, _)| d) == Some(*acc_id) + || carried_alias == Some(*acc_id) + { return None; } let mut facts = PureExprFacts { array: element_binding.map(|(_, arr_id)| arr_id), ..PureExprFacts::default() }; - if !element_shape_loop_pure_expr_collect(ctx, value, &scope, &mut facts) { + if !values + .iter() + .all(|value| element_shape_loop_pure_expr_collect(ctx, value, &scope, &mut facts)) + { return None; } let array_id = facts.array?; @@ -927,12 +1134,35 @@ fn match_element_shape_versioned_loop( return None; } } + // #10185: a matched recurrence whose value nothing subscripts with would + // still have its `LocalSet` lowered generically — a `frem` libcall inside + // the clone, which deletes it. Decline instead, so the loop keeps whatever + // lowering it has today. + match (carried.as_ref(), index) { + (Some(matched), MatchedIndex::Carried { .. }) => { + if matched.carried_id == array_id + || matched.modulus_id == array_id + || carried_alias == Some(array_id) + { + return None; + } + } + // A recurrence the subscripts do not use, or a carried subscript with + // no recurrence (which the walk's scope makes unreachable). + (Some(_), _) | (None, MatchedIndex::Carried { .. }) => return None, + (None, _) => {} + } match bound { ElementShapeLoopBound::Local(bound_id) => { if bound_id == array_id || bound_id == *acc_id || Some(bound_id) == scope.element_binding || derived.map(|(d, _)| d) == Some(bound_id) + // The bound is materialized ONCE in the preheader, so a bound + // the recurrence rewrites every iteration would be stale. The + // carried local is also not readable bare inside the clone. + || carried.is_some_and(|c| c.carried_id == bound_id) + || carried_alias == Some(bound_id) { return None; } @@ -975,12 +1205,6 @@ fn match_element_shape_versioned_loop( return None; } - for prop in &facts.props { - if CLASS_FIELD_LOOP_PROP_DENYLIST.contains(&prop.as_str()) { - return None; - } - } - let identity = match element_class_name(ctx, array_id, counter_id) { Some(class_name) => { // The class arm keeps its ORIGINAL grammar. Its bounds argument is @@ -999,6 +1223,42 @@ fn match_element_shape_versioned_loop( None => return None, }; + // The property denylist, which is arm-specific (#10185). + // + // The CLASS arm's list exists because its read bakes in a compile-time + // packed slot index while the surrounding lowering may route the name + // somewhere else entirely (`length` header loads, `errors` runtime calls, + // accessor-ish names) — a name collision there costs the call-free + // guarantee. + // + // The SHAPE arm bakes in nothing: the preheader asks the runtime for the + // inline slot of that exact key in that exact ordinary ShapeId and declines + // on `-1`, and the per-element residual pins `obj_type == GC_TYPE_OBJECT` + // with no per-object descriptors. Every receiver kind whose builtin branch + // could answer a name differently — a function's `name`, an array's + // `length`, a Map's `size`, an AggregateError's `errors` — fails that + // residual, and for a plain record an own data property SHADOWS the + // prototype name it collides with, which is exactly what the inline slot + // holds. The whole list would otherwise have cost the benchmark its + // `fields` shape outright, for a field literally called `name`. + // + // `__proto__` stays denied, and not because of JavaScript: node gives + // `JSON.parse('{"__proto__":1}')` an OWN `__proto__` data property and + // reads `1` back from it, so the inline slot would be right. It is denied + // because Perry's generic property path may special-case the name ahead of + // own-property lookup, and the clone must agree with the clone-of, not with + // the spec, wherever those two differ. + const SHAPE_PROP_DENYLIST: &[&str] = &["__proto__"]; + let denylist = match identity { + ElementShapeIdentity::Class { .. } => CLASS_FIELD_LOOP_PROP_DENYLIST, + ElementShapeIdentity::Shape => SHAPE_PROP_DENYLIST, + }; + for prop in &facts.props { + if denylist.contains(&prop.as_str()) { + return None; + } + } + // The declared accumulator type is only a candidate: the lowering // validates the accumulator's current NaN-box tag in the preheader before // installing the numeric fact for the fast clone. The class arm keeps its @@ -1018,6 +1278,40 @@ fn match_element_shape_versioned_loop( return None; } + // #10185: the fast clone's body, when the two rewrites apply. `lead` is + // whatever statements the walk consumed ahead of the accumulator run (the + // recurrence and/or the virtual binding), kept verbatim. + let lead = &body[..body.len() - rest.len()]; + let fast_body = (values.len() > 1 || carried.is_some()).then(|| { + let mut stmts: Vec = lead.to_vec(); + // `acc = v1; acc = acc p2; acc = acc p3` + // ≡ `acc = ((v1) p2) p3` + // — the same operations on the same operands in the same order, which + // is what keeps the folded float arithmetic bit-identical. + let mut folded = values[0].clone(); + for value in &values[1..] { + let Expr::Binary { op, right, .. } = value else { + unreachable!("the accumulator fold validated every statement's shape"); + }; + folded = Expr::Binary { + op: *op, + left: Box::new(folded), + right: right.clone(), + }; + } + stmts.push(Stmt::Expr(Expr::LocalSet(*acc_id, Box::new(folded)))); + if let Some(matched) = carried.as_ref() { + // The commit marker (`stmt/element_shape_carried.rs`). It is the + // LAST statement, so it runs only once the iteration is past every + // exit it could take. + stmts.push(Stmt::Expr(Expr::LocalSet( + matched.carried_id, + Box::new(Expr::LocalGet(matched.carried_id)), + ))); + } + stmts + }); + Some(ElementShapeVersionedLoop { counter_id, bound, @@ -1027,6 +1321,8 @@ fn match_element_shape_versioned_loop( index, element_binding: scope.element_binding, accumulator_id: *acc_id, + carried, + fast_body, }) } @@ -1200,10 +1496,25 @@ pub(super) fn lower_element_shape_versioned_for( // floor of 1 — `srem` by zero is undefined behaviour, and `x % 0` is NaN // in JS, so a zero modulus is a slow-clone case rather than something the // clone may compute. - let modulus_i32: Option = match matched.index { - MatchedIndex::DerivedMod { modulus_id, .. } => Some(materialize_loop_i32( + // + // #10185's carried recurrence takes the same obligation for the same + // reason, and one more of its own: its ENTRY value must be a non-negative + // integral i32, because the whole i64 recurrence bound + // (`a * i32::MAX + b < 2^53`, and a non-negative dividend for `srem` to + // agree with JS `%`) is stated over that range. A fractional, negative or + // non-number `cursor` at loop entry is a slow-clone case. + let modulus_i32: Option = match (&matched.index, &matched.carried) { + (MatchedIndex::DerivedMod { modulus_id, .. }, _) => Some(materialize_loop_i32( + ctx, + *modulus_id, + 1, + i32::MAX, + &slow_pre_label, + "element_shape.loop.modulus", + )?), + (MatchedIndex::Carried { .. }, Some(carried)) => Some(materialize_loop_i32( ctx, - modulus_id, + carried.modulus_id, 1, i32::MAX, &slow_pre_label, @@ -1211,6 +1522,25 @@ pub(super) fn lower_element_shape_versioned_for( )?), _ => None, }; + let carried_slot: Option = match &matched.carried { + Some(carried) => { + let entry = materialize_loop_i32( + ctx, + carried.carried_id, + 0, + i32::MAX, + &slow_pre_label, + "element_shape.loop.carried", + )?; + // Entry-block alloca: LLVM lowers a non-entry `alloca` as a real + // stack bump with no restore, and this one is written every + // iteration. + let slot = ctx.func.alloca_entry(I32); + ctx.block().store(I32, &entry, &slot); + Some(slot) + } + None => None, + }; let trip_count = match &materialized_bound { Some(bound) => { @@ -1225,11 +1555,14 @@ pub(super) fn lower_element_shape_versioned_for( (MatchedIndex::Constant(k), _) => { crate::expr::element_shape_guard::ElementShapeIndexBound::Constant(*k) } - (MatchedIndex::DerivedMod { .. }, Some(modulus)) => { + // #10185: the recurrence's result is `srem` of a non-negative dividend + // by `m`, so it lands in `[0, m)` exactly like the derived index and + // takes exactly the same preheader obligation, `m <= length`. + (MatchedIndex::DerivedMod { .. } | MatchedIndex::Carried { .. }, Some(modulus)) => { crate::expr::element_shape_guard::ElementShapeIndexBound::Modulus(modulus.as_str()) } - (MatchedIndex::DerivedMod { .. }, None) => { - unreachable!("a derived index always materializes its modulus") + (MatchedIndex::DerivedMod { .. } | MatchedIndex::Carried { .. }, None) => { + unreachable!("a modulo-derived index always materializes its modulus") } }; let expected_class_id_str = match &matched.identity { @@ -1323,8 +1656,58 @@ pub(super) fn lower_element_shape_versioned_for( // iteration. slot: ctx.func.alloca_entry(I32), }, + MatchedIndex::Carried { + carried_id, + alias_id, + } => { + let carried = matched + .carried + .as_ref() + .expect("a carried index always matches a recurrence"); + crate::expr::ElementShapeIndex::Carried(Box::new(crate::expr::CarriedIndex { + local_id: carried_id, + alias_id, + coeff_a: carried.coeff_a, + coeff_b: carried.coeff_b, + modulus_i32: modulus_i32 + .clone() + .expect("a carried index always materializes its modulus"), + slot: carried_slot + .clone() + .expect("a carried index always materializes its entry value"), + commit_slot: ctx + .locals + .get(&carried_id) + .cloned() + .expect("the matcher required a directly-addressable local"), + })) + } }; + // #10185: the shared once-per-iteration element deref, hung off the body's + // leading virtual statement. Shape-keyed only: the class-keyed arm's reads + // are raw doubles behind a `GC_OBJ_TYPED_LAYOUT_INTACT` residual whose + // emitted IR #7480's census pins, and nothing in #10185 measures it. + let elem_prefetch = shape_keyed + .then(|| { + let site_local_id = match (&fact_index, matched.element_binding) { + (_, Some(binding)) => Some(binding), + (crate::expr::ElementShapeIndex::DerivedMod { local_id, .. }, _) => Some(*local_id), + // With an alias the prologue belongs to the alias `Let`, which + // runs after the recurrence; without one it belongs to the + // recurrence itself. Exactly one statement owns it either way. + (crate::expr::ElementShapeIndex::Carried(carried), _) => { + Some(carried.alias_id.unwrap_or(carried.local_id)) + } + _ => None, + }?; + Some(crate::expr::ElementPrefetch { + site_local_id, + handle_slot: ctx.func.alloca_entry(I64), + }) + }) + .flatten(); + let scope_id = ctx.next_loop_proof_scope_id(); let fast_scan_start = ctx.func.num_blocks(); ctx.current_block = fast_pre_idx; @@ -1333,6 +1716,7 @@ pub(super) fn lower_element_shape_versioned_for( array_local_id: matched.array_id, index_local_id: matched.counter_id, index: fact_index, + elem_prefetch, shape_keyed, scope_id, class_name: report_class, @@ -1341,6 +1725,7 @@ pub(super) fn lower_element_shape_versioned_for( side_exit_label: slow_pre_label.clone(), statically_layout_proven, fields, + synthesized_body: matched.fast_body.is_some(), element_binding: matched.element_binding, numeric_accumulator: matched.accumulator_id, }); @@ -1349,7 +1734,7 @@ pub(super) fn lower_element_shape_versioned_for( init, condition, update, - body, + matched.fast_body.as_deref().unwrap_or(body), "for.element_shape_fast", Some((matched.counter_id, guard.bound_i32)), ); diff --git a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs index c78cfc72e5..20d71b035a 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop_tests.rs @@ -402,14 +402,34 @@ fn assert_fast_clone_is_entered(ir: &str) { fn fast_clone_slice(ir: &str) -> String { let mut owned = String::new(); let mut in_fast_block = false; + // #10185: the emitted text carries the function TWICE (same block labels), + // so every block would be collected twice and any `matches().count()` + // assertion against the slice would read double. Stop at the first repeated + // label, which is where the second copy begins — `contains` assertions are + // unaffected, and counting one iteration of the clone is what makes "the + // residual check happens ONCE" expressible at all. + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); for line in ir.split_inclusive('\n') { let trimmed = line.trim_end(); // A block DEFINITION starts at column 0 and ends in `:`; anything else // belongs to whichever block was last opened. if !line.starts_with(char::is_whitespace) && trimmed.ends_with(':') { + // #10185 added three more: the string-`.length` decode + // (`element_shape.strlen*`), the boolean ternary's admitted arm + // (`element_shape.bool`), and nothing for the shared prefetch, + // which reuses `element_shape.load`. Every block the clone can + // execute must be listed, or the negatives below go vacuous for it. in_fast_block = trimmed.starts_with("for.element_shape_fast.") || trimmed.starts_with("element_shape.load") - || trimmed.starts_with("element_shape.number"); + || trimmed.starts_with("element_shape.number") + || trimmed.starts_with("element_shape.strlen") + || trimmed.starts_with("element_shape.bool"); + // Only a repeated CLONE label means the second copy: unrelated + // functions share ordinary labels (`entry:`), and breaking on one + // of those would slice away the clone entirely. + if in_fast_block && !seen.insert(trimmed.to_string()) { + break; + } } if in_fast_block { owned.push_str(line); @@ -999,7 +1019,10 @@ fn object_literal_element_resolution_does_not_escape_the_clone() { .expect("the merge block should be DEFINED in the emitted IR")..]; assert!( after.contains("js_object_get_field_by_name_f64") - || after.contains("js_object_get_field_ic_miss"), + || after.contains("js_object_get_field_ic_miss") + // T1: the generic tower's cold arms are behind these two entries now. + || after.contains("js_object_get_field_ic_slow") + || after.contains("js_object_get_field_ic_nonptr"), "the post-loop read must stay on the by-name path; emitted:\n{after}" ); } @@ -1488,6 +1511,7 @@ fn assert_clone_fires_call_free(ir: &str, what: &str) { assert!( !fast.contains("js_object_get_field_by_name_f64") && !fast.contains("js_typed_feedback_class_field_get_guard") + && !fast.contains("js_class_field_get_ic") && !fast.contains("js_number_coerce"), "{what}: the by-name field diamond must be gone from the fast clone" ); diff --git a/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs b/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs index 94ade02cc7..6d5d265332 100644 --- a/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_shape_keyed_tests.rs @@ -450,21 +450,78 @@ fn an_untracked_local_index_declines() { } #[test] -fn a_denylisted_property_declines_the_shape_keyed_arm() { - // `length`, `name`, `constructor`, … are answered by the runtime or by the - // prototype, not out of an inline slot, so a shape's key position for one - // would be the wrong answer even when it exists. +fn the_shape_keyed_arm_denies_only_proto() { + // #10185 narrowed this list. The CLASS arm still denies every name with a + // dedicated branch in the property dispatch, because its read bakes in a + // compile-time packed slot. The SHAPE arm bakes in nothing: the preheader + // asks the runtime for that exact key's inline slot in that exact ordinary + // ShapeId and declines on `-1`, and the residual pins + // `obj_type == GC_TYPE_OBJECT` with no per-object descriptors — so every + // receiver whose builtin branch could answer a name differently (a + // function's `name`, an array's `length`, a Map's `size`) is already + // excluded, and for a plain record an own data property shadows the + // prototype name it collides with. Denying the whole list cost the access + // benchmark its `fields` shape outright, for a field called `name`. + for property in ["length", "name", "constructor", "size", "message"] { + let ir = emit(&untyped_param_module( + Type::Any, + Type::Number, + vec![untyped_accumulate(untyped_elem_field( + Expr::LocalGet(U_COUNTER_ID), + property, + ))], + )); + assert!( + ir.contains("element_shape.loop.fast.preheader"), + "`{property}` is an ordinary own inline slot on a parsed record; \ + the shape-keyed arm must serve it" + ); + } + // `__proto__` stays denied — not because of JavaScript (node gives + // `JSON.parse('{\"__proto__\":1}')` an own data property and reads `1` + // back), but because Perry's generic property path may special-case the + // name ahead of own-property lookup, and the clone must agree with the + // path it is a clone of. let ir = emit(&untyped_param_module( Type::Any, Type::Number, vec![untyped_accumulate(untyped_elem_field( Expr::LocalGet(U_COUNTER_ID), - "length", + "__proto__", + ))], + )); + assert!( + !ir.contains("element_shape.loop.fast.preheader"), + "`__proto__` must decline the shape-keyed clone" + ); +} + +#[test] +fn the_class_keyed_arm_keeps_the_full_property_denylist() { + // The narrowing above is shape-arm-only: a class-keyed read bakes in a + // packed slot index while the surrounding lowering may route the name + // somewhere else entirely, which is what the list has always protected. + let ir = emit(&element_shape_module( + vec![Stmt::Expr(Expr::LocalSet( + SUM_ID, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(SUM_ID)), + right: Box::new(Expr::PropertyGet { + object: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ARRAY_ID)), + index: Box::new(Expr::LocalGet(COUNTER_ID)), + }), + property: "length".to_string(), + byte_offset: 0, + }), + }), ))], + None, )); assert!( !ir.contains("element_shape.loop.fast.preheader"), - "a denylisted property must decline the clone" + "a denylisted property must still decline the CLASS-keyed clone" ); } @@ -489,3 +546,9 @@ fn a_declared_but_unresolvable_element_type_still_declines() { "an unresolvable declared element type must decline both arms" ); } + +/// #10185's `fields` and `random` shapes — a child module so it inherits both +/// this file's shape-keyed assertions and `element_shape_loop_tests`'s slicing +/// helpers. +#[path = "element_shape_fields_random_tests.rs"] +mod fields_random; diff --git a/crates/perry-codegen/src/stmt/let_stmt.rs b/crates/perry-codegen/src/stmt/let_stmt.rs index 28ec377674..288e799086 100644 --- a/crates/perry-codegen/src/stmt/let_stmt.rs +++ b/crates/perry-codegen/src/stmt/let_stmt.rs @@ -31,7 +31,7 @@ pub(crate) fn lower_let( // binding is VIRTUAL — the element binding emits nothing and the derived // index emits one `srem`. Both live with the clone, which is where their // soundness arguments and the preheader that validated them are. - if super::element_shape_loop::lower_virtual_clone_binding(ctx, id) { + if super::element_shape_loop::lower_virtual_clone_binding(ctx, id)? { return Ok(()); } // `let C = SomeClass` aliases the local `C` to the class diff --git a/crates/perry-codegen/src/stmt/mod.rs b/crates/perry-codegen/src/stmt/mod.rs index cc407d30c0..cc27d26763 100644 --- a/crates/perry-codegen/src/stmt/mod.rs +++ b/crates/perry-codegen/src/stmt/mod.rs @@ -20,6 +20,7 @@ mod cached_field_index_return; #[cfg(test)] mod class_field_loop_tests; mod counter_range; +mod element_shape_carried; mod element_shape_loop; #[cfg(test)] mod element_shape_loop_tests; @@ -281,6 +282,15 @@ fn lower_return_expr(ctx: &mut FnCtx<'_>, expr: &perry_hir::Expr) -> Result, stmt: &Stmt) -> Result<()> { match stmt { Stmt::Expr(e) => { + // #10185: the element-shape fast clone's carried-index statements + // (the recurrence and its trailing write-back) are lowered + // VIRTUALLY, exactly like the `Let` bindings in `let_stmt.rs` — + // the generic lowering of `c = (c * 17 + 7) % length` is an + // `frem` libcall, and a call inside the clone DELETES it (#7690). + // Outside a clone the fact vector is empty and this is a no-op. + if element_shape_carried::lower_virtual_carried_stmt(ctx, e)? { + return Ok(()); + } let prev_discard = ctx.discard_expr_value; ctx.discard_expr_value = true; // #7590: the non-leaking companion. `lower_expr` takes this at the diff --git a/crates/perry-codegen/src/type_analysis/numeric.rs b/crates/perry-codegen/src/type_analysis/numeric.rs index e3677034e2..338442e398 100644 --- a/crates/perry-codegen/src/type_analysis/numeric.rs +++ b/crates/perry-codegen/src/type_analysis/numeric.rs @@ -356,6 +356,18 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { Expr::Logical { left, right, .. } => { is_numeric_expr(ctx, left) && is_numeric_expr(ctx, right) } + // #10185: `arr[i]. ? 1 : 0` inside an element-shape fast + // clone. Deliberately NOT the general "both arms are numeric" rule — a + // general ternary's CONDITION is an arbitrary JS truthiness test, which + // is a runtime call; this arm is true only for the exact shape the + // clone lowers itself, where the condition is a tracked element read + // admitted as one of the two boolean singletons and everything else + // side-exits. + Expr::Conditional { .. } + if crate::expr::element_shape_reads::cloned_bool_select_read(ctx, e).is_some() => + { + true + } // `obj.field` where the field is declared as `number` on the // owning class. Without this, `this.value + 1` in a hot loop // wraps the field load in `js_number_coerce` which prevents @@ -374,6 +386,17 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { if property == "length" && expression_has_numeric_length(ctx, object) { return true; } + // #10185: `arr[i]..length` inside an element-shape + // fast clone. The receiver is an untyped element read, so the + // declared-type answer above cannot see a string there; the clone's + // own lowering tag-tests the loaded word for both string + // representations and side-exits when it is not one, which is a + // stronger proof than an annotation. Without this the `+` consuming + // it bails to `js_dynamic_string_or_number_add` and the call + // deletes the clone. + if crate::expr::element_shape_reads::cloned_string_length_read(ctx, e).is_some() { + return true; + } // repsel #7480 step 3: inside an element-shape fast clone a tracked // `arr[i].field` read is a GUARD-PROVEN raw double. The class-keyed // arm gets that from the residual check's diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index b14de19aa9..69480ae817 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -12843,8 +12843,15 @@ fn typed_f64_receiver_method_clone_raw_loads_after_composed_guards() { // about which emission shape carried it. let inline_probe = caller_ir.find("@PERRY_CLASS_PROTOTYPE_FAST_GUARDS_INVALIDATED"); let method_proof = inline_probe.map_or(method_guard, |p| p.min(method_guard)); + // One-exit class-field GET: at a field-GET site the runtime guard is no + // longer CALLED -- it is the body of `js_class_field_get_ic`, the inline + // pre-check's single miss edge. Other lowerings (the method-override + // receiver-clone gate) still call the guard directly, so accept either + // witness; what this assertion is about is that a field proof exists and + // dominates the typed call, not which symbol carries it. let field_guard = caller_ir - .find("call i32 @js_typed_feedback_class_field_get_guard") + .find("call double @js_class_field_get_ic") + .or_else(|| caller_ir.find("call i32 @js_typed_feedback_class_field_get_guard")) .unwrap_or_else(|| panic!("caller should guard raw-f64 receiver fields:\n{caller_ir}")); // Same for the raw-f64 FIELD proof: one inline class-field precheck // (the field-GET sites' form, first mentioned by its `deref` block in the @@ -14866,7 +14873,11 @@ fn annotated_class_method_value_uses_generic_lookup() { // fallback, which is the exact regression #8033 exists to prevent. let generic = ir_function_body(&ir, "__probe$generic("); assert!( - generic.contains("call double @js_object_get_field_ic_miss"), + // T1 renamed the tower's cold exits; this assertion is about the + // unguarded body keeping GENERIC lookup, not about which symbol + // serves it. + generic.contains("call double @js_object_get_field_ic_slow") + || generic.contains("call double @js_object_get_field_ic_miss"), "an annotation-only class receiver must preserve generic property \ lookup in the unguarded body:\n{generic}" ); diff --git a/crates/perry-codegen/tests/native_proof_regressions/pod_manifest.rs b/crates/perry-codegen/tests/native_proof_regressions/pod_manifest.rs index 77222de320..d119a37e2c 100644 --- a/crates/perry-codegen/tests/native_proof_regressions/pod_manifest.rs +++ b/crates/perry-codegen/tests/native_proof_regressions/pod_manifest.rs @@ -652,7 +652,11 @@ fn native_pod_view_length_survives_immutable_local_alias() { "an immutable PodView alias must use the validating length helper:\n{ir}" ); assert!( - !ir.contains("call double @js_object_get_field_ic_miss"), + // T1 renamed the tower's exits; a negative assertion that names only + // the retired symbol can no longer fail, so it names the live ones. + !ir.contains("call double @js_object_get_field_ic_slow") + && !ir.contains("call double @js_object_get_field_ic_nonptr") + && !ir.contains("call double @js_object_get_field_ic_miss"), "a PodView alias must not enter the ordinary object-property PIC:\n{ir}" ); } diff --git a/crates/perry-codegen/tests/typed_feedback.rs b/crates/perry-codegen/tests/typed_feedback.rs index 3ee8f733eb..557ad75720 100644 --- a/crates/perry-codegen/tests/typed_feedback.rs +++ b/crates/perry-codegen/tests/typed_feedback.rs @@ -581,8 +581,9 @@ fn a_default_build_emits_no_typed_feedback_recording_calls() { // empty string. The property boundaries themselves must still be here — // this test proves the RECORDING is gone, not the program. assert!( - ir.contains("js_object_get_field_by_name_f64") - || ir.contains("js_object_get_field_ic_miss"), + ir.contains("call double @js_object_get_field_ic_slow(") + || ir.contains("call double @js_object_get_field_by_name_f64(") + || ir.contains("call double @js_object_get_field_ic_miss"), "the property reads themselves must still be lowered; emitted:\n{ir}" ); // And the helpers that DECIDE something, rather than merely counting, are @@ -598,8 +599,15 @@ fn a_default_build_emits_no_typed_feedback_recording_calls() { // the two dispatchers this same fixture still emits -- the property GET // (the set dispatcher's twin) and the method call -- and as CALLS, since // the old symbol match was satisfied by the `declare` line alone. + // T1: the property GET's dispatching wrapper is one indirection further + // out. `js_typed_feedback_object_get_field_by_name_f64` is no longer + // emitted per site — it is the INT32 class-ref arm of + // `js_object_get_field_ic_nonptr`, which the site calls with the same + // `site_id`. The line this assertion draws is unchanged: a dispatcher that + // DECIDES something is emitted in a default build, a helper that merely + // counts is not (all six are asserted absent above). assert!( - ir.contains("call double @js_typed_feedback_object_get_field_by_name_f64("), + ir.contains("call double @js_object_get_field_ic_nonptr("), "dispatching feedback wrappers must still be emitted in a default build \ (property get):\n{ir}" ); @@ -646,20 +654,28 @@ fn typed_feedback_guards_direct_class_field_specialization() { )); assert!(ir.contains("class_field_set_guard")); - assert!(ir.contains("class_field_get_guard")); assert!(ir.contains("@perry_typed_shape_raw_f64_mask_")); assert!(ir.contains("js_typed_feedback_class_field_set_guard")); - assert!(ir.contains("js_typed_feedback_class_field_get_guard")); assert!(ir.contains("class_field_set.fast")); assert!(ir.contains("class_field_set.fallback")); // #8033: `receiver_class_name` now consults `stable_local_type_proof` // (runtime evidence), which a bare parameter lacks. The numeric-specific // `class_field_get_number.*` path is therefore not selected; the generic - // `class_field_get.*` diamond is emitted instead, with its own fast/fallback - // arms and the typed-feedback guard. The guard assertions above (lines - // 493-499) validate the test's core purpose. + // `class_field_get.*` tower is emitted instead. + // + // One-exit class-field GET: the tower keeps the #5093 inline pre-check and + // the fast slot load, and everything behind the pre-check — the guard call, + // the guard-PASS load, the nullish TypeError and the by-name lookup — is + // now the body of `js_class_field_get_ic`. The guard therefore still runs + // on every miss, one frame deeper; asserting `js_typed_feedback_class_ + // field_get_guard` as a bare substring here would prove nothing either way, + // because the `declare` line alone satisfies it (the same trap #7480's + // comment below describes). Assert the call FORMS instead. assert!(ir.contains("class_field_get.fast")); - assert!(ir.contains("class_field_get.fallback")); + assert!(ir.contains("class_field_inline.deref")); + assert!(ir.contains("call double @js_class_field_get_ic(")); + assert!(!ir.contains("call i32 @js_typed_feedback_class_field_get_guard(")); + assert!(!ir.contains("class_field_get.fallback")); assert!(ir.contains("store double")); assert!(!ir.contains("call void @js_gc_note_slot_layout")); // #5334 lever A: the SET fallback arm collapses to one outlined call; the @@ -670,9 +686,13 @@ fn typed_feedback_guards_direct_class_field_specialization() { // (from the class-field-GET fallback block, not the SET site — the SET copy // was folded into js_class_field_set_fallback by #5334). It is now emitted // only in a typed-feedback build, like the registration call beside it. - // The fallback ARM itself is unchanged and is asserted above/below. + // The fallback ARM itself is asserted above. assert!(!ir.contains("call void @js_typed_feedback_record_fallback_call")); - assert!(ir.contains("call double @js_object_get_field_by_name_f64")); + // The GET miss arm's by-name lookup moved inside `js_class_field_get_ic` + // (which records the fallback call itself, under the same runtime + // `typed_feedback_enabled()` gate the elided emission above respects), so + // the site no longer emits it. + assert!(!ir.contains("call double @js_object_get_field_by_name_f64(")); } /// Body of the first rendered block whose label starts with `label_prefix`, @@ -774,23 +794,27 @@ fn full_outline_ic_collapses_class_field_get_to_single_call() { let _lock = env_lock(); - // Forced ON: one outlined call, no inline get diamond. + // Forced ON: one outlined call and NO tower at all -- not even the #5093 + // inline pre-check, which is the whole point on an oversized module. { let _g = EnvVarGuard::set("PERRY_FULL_OUTLINE_IC", Some("1")); let ir = ir_for(build()); assert!(ir.contains("call double @js_class_field_get_ic")); assert!(!ir.contains("class_field_get.fast")); - assert!(!ir.contains("class_field_get.fallback")); - assert!(!ir.contains("call i32 @js_typed_feedback_class_field_get_guard")); + assert!(!ir.contains("class_field_inline.deref")); + assert!(!ir.contains("call i32 @js_typed_feedback_class_field_get_guard(")); } - // Forced OFF: the inline diamond, no full-outline call. + // Forced OFF: the inline tower. Since the one-exit change, BOTH arms call + // `js_class_field_get_ic`, so the discriminator is the TOWER, not the call: + // OFF keeps the pre-check and the fast slot load, ON has neither. { let _g = EnvVarGuard::set("PERRY_FULL_OUTLINE_IC", Some("0")); let ir = ir_for(build()); - assert!(!ir.contains("call double @js_class_field_get_ic")); + assert!(ir.contains("class_field_inline.deref")); assert!(ir.contains("class_field_get.fast")); - assert!(ir.contains("js_typed_feedback_class_field_get_guard")); + assert!(ir.contains("call double @js_class_field_get_ic")); + assert!(!ir.contains("call i32 @js_typed_feedback_class_field_get_guard(")); } } diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 318bb493d9..dc9438fcd3 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -309,8 +309,14 @@ pub use ic_miss::{ js_object_set_field_by_property_id, js_private_brand_add, js_private_brand_check, js_private_field_add, js_private_guard, PicCache, PicCacheSlot, PIC_CACHE_WORDS, }; +/// The one slow exit of the emitted generic property-get tower. Declared here +/// rather than inside `ic_miss.rs` only because that file sits at the +/// 2000-line cap; the source lives next to its sibling entries. +#[path = "field_get_set/ic_miss/ic_slow.rs"] +mod ic_slow; pub(crate) use ic_slot::pic_slot_census; pub use ic_slot::{pic_arena_bytes, pic_slot_peek, pic_slot_resolve, pic_slots_resolved}; +pub use ic_slow::{js_object_get_field_ic_nonptr, js_object_get_field_ic_slow}; #[cfg(test)] mod buffer_ic_miss_tests { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index 41a847b423..9cedc25adb 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -537,7 +537,7 @@ fn ic_diag_note( mod packed_get; pub use packed_get::{js_object_get_field_ic_miss, js_object_get_field_ic_miss_packed}; -fn get_field_ic_miss_impl( +pub(super) fn get_field_ic_miss_impl( obj: *const ObjectHeader, key: *const crate::StringHeader, cache_slot: *mut PicCacheSlot, diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs new file mode 100644 index 0000000000..00d62f64d8 --- /dev/null +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -0,0 +1,585 @@ +//! The two slow exits of the emitted generic property-get tower. +//! +//! # Why this exists +//! +//! `lower_generic_property_get` used to emit ~37 basic blocks and 6-7 runtime +//! call sites per untyped `obj.prop`: an SSO arm, an INT32 class-ref arm, a +//! nullish-throw arm, a non-object-receiver arm, an overflow-slot load, a +//! deleted-slot miss, two Array-subclass named-prefix ladders, and the +//! miss+prime. Every one of those calls is a statepoint, so both the emitted +//! `.text` and `.perry_gcmap` scale with call sites x live GC values — on +//! @babel/parser that tower was 29% of all emitted IR over 6,487 sites. +//! +//! The inline *hit* is worth its bytes; the arms around it are not. These two +//! entries are what the site branches to instead of each of them, reproducing +//! the arms they replaced in the same ORDER and with the same cache-priming +//! decisions. The emitted site keeps, byte for byte, the receiver-tag test, the +//! small-handle test, the packed header kind/descriptor test, the packed-MRU +//! compare, the overflow-bit test, the inline field load with its hole check, +//! and the polymorphic ways. +//! +//! # Why TWO exits and not one +//! +//! The first cut had one entry taking the unmasked NaN-box, reached from every +//! guard failure including the receiver-tag test. It cost **+4.00 instructions +//! on every property-read HIT**, measured on a monomorphic 10M-read loop +//! (1,231,521,261 -> 1,271,522,669 retired instructions, +3.2%), and the +//! disassembly says exactly why: with the tag test and the small-handle test +//! failing to the SAME block, LLVM's SimplifyCFG folds them into one flat +//! predicate — +//! +//! ```text +//! cmp …; sete %dl; cmp …; setae %dil; test %dil,%dl; je (6 instructions) +//! ``` +//! +//! — where the branchy chain was `cmp; jne; cmp; ja` (4). That is #7883's +//! finding re-introduced from the other side: there codegen flattened the +//! guards, here the optimiser does it because the shared exit made the two +//! branches congruent. Giving the tag test its own callee restores the chain +//! (and, as a bonus, lets the unmasked `obj_bits` die in the entry block +//! instead of staying live across the whole hit path for a call it might make). +//! +//! The split is along the one line that matters: [`js_object_get_field_ic_nonptr`] +//! serves receivers that are NOT heap pointers (and needs the tag, not the +//! cache), [`js_object_get_field_ic_slow`] serves the ones that are (and needs +//! the cache, not the tag). Two call sites per site instead of six. +//! +//! # The arms, in the order the emitted tower took them +//! +//! 1. Receiver-tag routing ([`js_object_get_field_ic_nonptr`]), exactly as +//! `js_object_get_field_ic` does it (SSO -> the SSO-aware by-name helper; +//! INT32 class ref -> the feedback-wrapped by-name helper; nullish -> +//! `TypeError`; any other non-pointer -> the by-name helper, which still +//! resolves typed shapes such as Date `.constructor`). +//! 2. A packed-MRU token HIT that the emitted hit path declined: +//! * the slot carries `IC_SLOT_OVERFLOW_BIT` — the field lives in the spill +//! buffer, so the inline `obj + header + slot*8` arithmetic must not run +//! on it. This is `js_object_get_field_ic_overflow_load`'s body, including +//! its fallback to the miss handler **without** the packed republication +//! (that helper called the three-argument `js_object_get_field_ic_miss`, +//! and re-publishing the packed word here would change which arm the next +//! read of the same site takes); +//! * otherwise the inline load found `TAG_HOLE` — a field deleted since +//! priming — and takes the ordinary miss, with the packed word, exactly +//! as the emitted `pic.hit.deleted` -> `pic.miss.call` edge did. +//! 3. The Array-subclass named-prefix proof (cache word 2 against `ObjectMeta` +//! word 6, then the cached slot). The emitted tower had this twice, once for +//! a descriptor-free receiver whose exact ShapeId missed and once for a +//! descriptor-bearing one that could never take the raw-load PIC at all; +//! both reduce to the same three loads on a `GC_TYPE_OBJECT` receiver with a +//! resolved cache, so they are one arm here. +//! 4. Everything else: `get_field_ic_miss_impl`, which owns the priming policy +//! (`prime_get`, the megamorphic countdown, the `PERRY_IC_DIAG` rows). +//! +//! # What is deliberately NOT here +//! +//! The polymorphic ways stay inline. They are read-only compares against a +//! cache the site already resolved, they serve the receiver rotation that is +//! the whole point of #7753, and moving them behind a call is a separate +//! measurement. +//! +//! Typed feedback is unchanged: the OBSERVE call still sits inline in +//! `pget.recv_ok` and the guard-pass / guard-fail / fallback-call records still +//! sit on the emitted edges, all under the same compile-time gate. The one +//! feedback call that lives here is the class-ref arm's, which was a +//! feedback-wrapped helper on that edge before and still is. +//! +//! # Rooting +//! +//! Every call these functions make is in TAIL position: the tag-routing arms, +//! the overflow fallback, and `get_field_ic_miss_impl` are all `return`s, and +//! the two probes above them (`overflow_get`, the named-prefix loads) neither +//! allocate nor enter user code. There is therefore no window in which a raw +//! `obj`/`key` argument is named after a collection point, which is the +//! condition a `RuntimeHandleScope` exists to cover (cf. `js_put_value_set_ic_miss`, +//! which re-reads its key to prime AFTER the allocating `js_put_value_set`). +//! Adding a scope here would root nothing and would put a TLS push/pop on every +//! megamorphic read. If an arm ever grows work after a call, it needs the scope +//! at that moment. + +use crate::object::{ObjectHeader, PicCacheSlot}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Cache word 2: the optional Array-subclass class-declared named-prefix +/// token. Mirrors `PIC_NAMED_PREFIX_TOKEN` in +/// `perry-codegen/src/expr/property_get/generic_dispatch.rs`. +const PIC_NAMED_PREFIX_TOKEN: usize = 2; + +/// The exit for a receiver that is NOT a heap pointer: the emitted site's +/// `(tag & 0xFFFD) == 0x7FFD` test just failed. +/// +/// It takes no cache: none of these arms can prime one, which is exactly why +/// splitting them off costs nothing and buys the emitted guard chain back (see +/// the module header). +/// +/// * `obj_bits` — the receiver's full, UNMASKED NaN-box bits. +/// * `key` — the interned property-name `StringHeader`, already masked. +/// * `site_id` — the typed-feedback site id, used only by the class-ref arm. +#[no_mangle] +pub extern "C" fn js_object_get_field_ic_nonptr( + obj_bits: i64, + key: *const crate::StringHeader, + site_id: u64, +) -> f64 { + let bits = obj_bits as u64; + let tag = bits >> 48; + let obj_unmasked = bits as usize as *const ObjectHeader; + + // SSO receiver (SHORT_STRING_TAG): the SSO-aware by-name helper reads + // `.length` from the NaN-box payload and answers undefined otherwise. + // A `.length` site keeps serving this inline and never gets here. + if tag == 0x7FF9 { + return super::js_object_get_field_by_name_f64(obj_unmasked, key); + } + // INT32-tagged class ref: static field / dynamic IIFE-set property / + // synthetic `constructor`. Passes the UNMASKED bits so the runtime can + // detect the tag, exactly as the emitted `pget.recv_class_ref` did. + if tag == 0x7FFE { + return crate::typed_feedback::js_typed_feedback_object_get_field_by_name_f64( + site_id, + obj_unmasked, + key, + ); + } + // `undefined`/`null` throw a node-shaped TypeError (#462). The emitted + // arm passed the property's static bytes and ended in `unreachable`; + // this reads the same bytes off the interned key and the helper is + // `-> !`, so the two are the same divergence with the same message. + if bits == crate::value::TAG_UNDEFINED || bits == crate::value::TAG_NULL { + let is_null = u32::from(bits == crate::value::TAG_NULL); + let (ptr, len) = unsafe { + match super::super::has_own_helpers::str_from_string_header(key) { + Some(s) => (s.as_ptr(), s.len()), + None => (std::ptr::null(), 0), + } + }; + crate::error::js_throw_type_error_property_access(is_null, ptr, len); + } + // Every other tag: no auto-boxing, but the by-name helper still recognizes + // typed shapes (Date `.constructor` via DATE_REGISTRY). A POINTER/STRING + // receiver never reaches this entry from emitted code; if one ever did, + // this arm is the correct — merely non-priming — answer for it too, so + // there is no unverified mode behind the split. + super::js_object_get_field_by_name_f64(obj_unmasked, key) +} + +/// The spill-buffer read, out of line. +/// +/// Kept in its own `#[cold] #[inline(never)]` function for a code-shape reason +/// that is worth 12 instructions on EVERY slow call: inlined, its +/// `overflow_get` call forces `js_object_get_field_ic_slow` to save +/// callee-saved registers across it, which gives that function a real frame +/// (`push rbp/r15/r14/rbx` + the matching pops) and stops the miss handler from +/// being a sibling call. Out of line, the entry's every call is in tail +/// position and it needs no frame at all. +/// +/// Semantics are `js_object_get_field_ic_overflow_load`'s, unchanged: read +/// through `overflow_get`, and on a tombstoned slot fall back to the +/// THREE-argument miss entry — i.e. with a null `packed`, because +/// republishing the packed word here would re-decide which arm the next read +/// of this site takes. +/// +/// # Safety +/// Same contract as the caller: `obj` is a live `GC_TYPE_OBJECT` above the +/// handle band whose shape stamp matched the packed word, and `slot` is that +/// word's high half with `IC_SLOT_OVERFLOW_BIT` set. +#[cold] +#[inline(never)] +unsafe fn overflow_arm( + obj: *const ObjectHeader, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + slot: u32, +) -> f64 { + let idx = (slot & !crate::proxy::IC_SLOT_OVERFLOW_BIT) as usize; + if let Some(v) = crate::object::overflow_get(obj as usize, idx) { + if v != crate::value::TAG_HOLE { + return f64::from_bits(v); + } + } + super::ic_miss::get_field_ic_miss_impl(obj, key, cache_slot, std::ptr::null()) +} + +/// The exit for a receiver that IS a heap pointer — every failing guard on the +/// emitted site's object path lands here. +/// +/// * `obj_handle` — the receiver with the NaN-box tag already masked off. The +/// caller has established the POINTER/STRING tag; this entry re-establishes +/// everything below it, because a guard failure says nothing about WHICH +/// guard failed. +/// * `key` — the interned property-name `StringHeader`, already masked. +/// * `cache_slot` — the site's [`PicCacheSlot`] (`@perry_ic_N`), possibly still +/// null: `pic_slot_peek` answers null and the miss handler resolves it when +/// it actually primes. +/// * `packed` — the site's compact MRU word (`@perry_ic_N_packed_get`). +#[no_mangle] +pub extern "C" fn js_object_get_field_ic_slow( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, +) -> f64 { + let obj = obj_handle as usize as *const ObjectHeader; + let addr = obj as usize; + // Below the handle band the emitted code never dereferenced, and neither + // does this: the miss handler routes registry ids through + // HANDLE_PROPERTY_DISPATCH. + if crate::value::addr_class::is_above_handle_band(addr) { + // SAFETY: the caller's emitted `(tag & 0xFFFD) == 0x7FFD` test + // established a POINTER/STRING NaN-box and the address is above the + // handle band — the same licence under which the emitted tower loaded + // this header word inline. + unsafe { + let header = &*((addr - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader); + if header.obj_type == crate::gc::GC_TYPE_OBJECT { + let plain = header._reserved & crate::gc::OBJ_FLAG_HAS_DESCRIPTORS == 0; + // --- 2. the MRU token hit the emitted hit path declined ----- + if plain && !packed.is_null() { + // The emitted guard is `icmp eq i32 %pcid, trunc(%packed)` + // against the RAW header word, with no range check, and + // this is that test: a nonzero packed word always carries a + // valid ShapeId (`prime_get` publishes nothing else), and a + // receiver whose word is an ordinary class id can never + // equal one, so `object_shape_stamp`'s range check would + // only re-derive what the equality already proves (#809's + // keyless receiver fails the equality, not the range test). + let word = (*packed).load(Ordering::Relaxed); + if word != 0 && (*obj).parent_class_id == word as u32 { + let slot = (word >> 32) as u32; + if slot & crate::proxy::IC_SLOT_OVERFLOW_BIT != 0 { + return overflow_arm(obj, key, cache_slot, slot); + } + // An inline slot that reached this entry on a token hit + // was a `TAG_HOLE` — the field was deleted since + // priming. The emitted `pic.hit.deleted` edge took the + // ordinary miss WITH the packed word. + return super::ic_miss::get_field_ic_miss_impl( + obj, key, cache_slot, packed, + ); + } + } + // --- 3. the Array-subclass named-prefix proof -------------- + // An object-backed Array subclass mints a new exact ShapeId on + // every numeric push/pop, but its class-declared named prefix + // does not move. The runtime proves the complete prefix once + // and publishes one token in cache word 2 and in ObjectMeta; + // any structural or descriptor transition clears the owner's. + // + // The conjunction is ordered by what it costs to ASK, not by + // which half the emitted tower read first: `meta` is one load + // from the header line this function has already touched + // (`obj - 8` and `obj + 4`), while the cache word is two + // dependent loads through the site's slot. An object with no + // metadata record cannot carry the token, so testing it first + // retires the overwhelmingly common non-subclass receiver in + // two instructions instead of eight — worth 6 on every miss, + // and the same conjunction either way. + let meta = (*obj).meta; + if !meta.is_null() { + let cache = crate::object::pic_slot_peek(cache_slot); + if !cache.is_null() + && (*cache)[PIC_NAMED_PREFIX_TOKEN] != 0 + && (*meta).array_subclass_named_prefix_token + == (*cache)[PIC_NAMED_PREFIX_TOKEN] as u64 + { + // Word 1 is an INLINE slot whenever word 2 is armed: an + // overflow prime writes word 2 = 0 precisely so this + // address arithmetic can never see one. + let slot = (*cache)[1] as usize; + let field = (obj as *const u8) + .add(std::mem::size_of::() + slot * 8) + as *const f64; + return *field; + } + } + } + } + } + + // --- 4. everything else: the miss handler owns the priming policy ------- + super::ic_miss::get_field_ic_miss_impl(obj, key, cache_slot, packed) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::object::{PicCache, PIC_CACHE_WORDS}; + + fn key_of(bytes: &[u8]) -> *const crate::StringHeader { + crate::string::js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32) + } + + /// What the emitted site passes: the NaN box with its tag masked off. The + /// tag itself was already tested inline, which is why this entry never + /// sees it. + fn handle(obj: *mut ObjectHeader) -> i64 { + (obj as u64 & 0x0000_FFFF_FFFF_FFFF) as i64 + } + + /// A plain own data read that has never primed: the entry must fall all the + /// way through to the miss handler, answer the field, and leave the site + /// primed exactly as the old `js_object_get_field_ic_miss_packed` edge did. + #[test] + fn plain_miss_answers_the_field_and_primes_both_caches() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + let key = scope.root_string_ptr(key_of(b"ic_slow_plain")); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|k| crate::object::js_object_set_field_by_name(o, k, 7.0)) + }); + + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let v = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(v, 7.0); + assert_ne!(cache[0], 0, "the full cache must be primed by the miss"); + assert_eq!(cache[1], 0, "the first own data field lives in slot 0"); + let word = packed.load(Ordering::Relaxed); + assert_ne!(word, 0, "the compact MRU must be published too"); + assert_eq!(word >> 32, 0, "slot 0, with no overflow bit"); + } + + /// An SSO receiver can never be dereferenced as an ObjectHeader. `.length` + /// is served from the NaN-box payload; any other key answers undefined. + #[test] + fn sso_receiver_routes_to_the_by_name_helper() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let sso = crate::value::JSValue::try_short_string(b"hey").unwrap(); + let key = scope.root_string_ptr(key_of(b"length")); + let v = key.with_const_ptr(|k| js_object_get_field_ic_nonptr(sso.bits() as i64, k, 0)); + assert_eq!(v, 3.0, "the SSO length byte"); + let other = scope.root_string_ptr(key_of(b"nope_not_here")); + let v = other.with_const_ptr(|k| js_object_get_field_ic_nonptr(sso.bits() as i64, k, 0)); + assert_eq!(v.to_bits(), crate::value::TAG_UNDEFINED); + } + + /// The overflow arm, both directions. + /// + /// A field past the inline region primes with `IC_SLOT_OVERFLOW_BIT` + /// (#9287), which is the exact state that sends the emitted MRU hit here + /// instead of doing the `obj + header + slot*8` arithmetic — that address + /// is not where the value lives. The positive direction is the spill read; + /// the negative one is the tombstone, where this entry must fall back to + /// the full miss handler rather than answer the hole. + #[test] + fn an_overflow_slot_reads_through_the_spill_and_falls_back_when_tombstoned() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + // INLINE_SLOT_FLOOR is 2, so the fourth key is past the inline region. + let mut keys = Vec::new(); + for (i, name) in [ + b"ic_slow_ovf_a".as_slice(), + b"ic_slow_ovf_b".as_slice(), + b"ic_slow_ovf_c".as_slice(), + b"ic_slow_ovf_d".as_slice(), + ] + .iter() + .enumerate() + { + let k = scope.root_string_ptr(key_of(name)); + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, i as f64)) + }); + keys.push(k); + } + let key = keys.last().expect("four keys"); + + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let read = |slot: &mut PicCacheSlot, packed: &AtomicU64| { + obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, slot, packed)) + }) + }; + assert_eq!(read(&mut slot, &packed), 3.0, "the priming read"); + let word = packed.load(Ordering::Relaxed); + assert_ne!(word, 0, "test premise: the site primed"); + assert_ne!( + (word >> 32) as u32 & crate::proxy::IC_SLOT_OVERFLOW_BIT, + 0, + "test premise: the fourth field must live past the inline region, \ + or this test never reaches the overflow arm (packed word {word:#x})" + ); + // The hit the emitted code now delegates: same packed pair, read + // through `overflow_get`. + assert_eq!(read(&mut slot, &packed), 3.0, "the overflow-arm read"); + + // Tombstone it. The spill slot no longer holds a value, so the arm must + // decline and let the full miss handler answer for the prototype chain. + obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| { + assert_eq!(crate::object::js_object_delete_field(o, k), 1); + }) + }); + assert_eq!( + read(&mut slot, &packed).to_bits(), + crate::value::TAG_UNDEFINED, + "a tombstoned overflow slot must miss, not answer the hole" + ); + } + + /// A non-nullish, non-pointer receiver keeps the old fall-through: no + /// throw, no cache, `undefined` unless the runtime knows the typed shape. + #[test] + fn plain_number_receiver_returns_undefined_without_priming() { + let _lock = crate::gc::global_side_table_test_lock(); + let key = key_of(b"ic_slow_on_a_double"); + let v = js_object_get_field_ic_nonptr(3.5f64.to_bits() as i64, key, 0); + assert_eq!(v.to_bits(), crate::value::TAG_UNDEFINED); + } + + /// The nullish arm must reach `js_throw_type_error_property_access` with + /// the key's bytes. `js_throw` exits the process when `TRY_DEPTH == 0`, so + /// the assertion here is on the pre-throw routing decision: `undefined` and + /// `null` are the only two bit patterns that take it, which is what the + /// emitted `pget.throw_nullish` predicate tested. + #[test] + fn only_undefined_and_null_take_the_throwing_arm() { + for bits in [crate::value::TAG_UNDEFINED, crate::value::TAG_NULL] { + assert!( + (bits >> 48) & 0xFFFD != 0x7FFD, + "a nullish tag must not look like a heap pointer" + ); + } + // TAG_FALSE/TAG_TRUE sit in the same 0x7FFC band and must NOT throw. + let key = key_of(b"ic_slow_on_a_bool"); + let v = js_object_get_field_ic_nonptr(crate::value::TAG_TRUE as i64, key, 0); + assert_eq!(v.to_bits(), crate::value::TAG_UNDEFINED); + } + + /// A hole in the primed inline slot (the field was deleted) must take the + /// ordinary miss and answer what the full lookup answers — never the raw + /// `TAG_HOLE` word. + #[test] + fn a_deleted_inline_slot_takes_the_ordinary_miss() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + let key = scope.root_string_ptr(key_of(b"ic_slow_deleted")); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|k| crate::object::js_object_set_field_by_name(o, k, 5.0)) + }); + + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let first = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(first, 5.0); + assert_ne!(packed.load(Ordering::Relaxed), 0, "test premise: primed"); + + obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| { + assert_eq!(crate::object::js_object_delete_field(o, k), 1); + }) + }); + let after = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!( + after.to_bits(), + crate::value::TAG_UNDEFINED, + "a deleted field reads undefined, not the hole word" + ); + } + + /// The named-prefix arm: an armed cache word 2 that matches the receiver's + /// ObjectMeta token serves the cached slot WITHOUT consulting the exact + /// ShapeId — that is the whole point of the proof, since an object-backed + /// Array subclass re-shapes on every numeric write. + /// + /// Driven directly against the two words the emitted ladder read, so it + /// pins this entry's contract with codegen rather than the subclass + /// machinery that publishes the token. + #[test] + fn an_armed_named_prefix_serves_the_cached_slot() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + let key = scope.root_string_ptr(key_of(b"ic_slow_prefix")); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|k| crate::object::js_object_set_field_by_name(o, k, 11.0)) + }); + // Give the receiver a meta record and an arbitrary prefix token. + const TOKEN: u64 = 0xA11CE; + // Collections are suppressed around the allocating meta mint, so the + // receiver cannot move while the scoped pointer is live. + let meta = obj.with_mut_ptr(|o: *mut ObjectHeader| { + let _no_gc = crate::gc::GcSuppressScope::new(); + unsafe { crate::object::object_meta_ensure(o) } + }); + assert!( + !meta.is_null(), + "test premise: the receiver has a meta record" + ); + unsafe { (*meta).array_subclass_named_prefix_token = TOKEN }; + + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + cache[PIC_NAMED_PREFIX_TOKEN] = TOKEN as i64; + cache[1] = 0; // the field's inline slot + // Word 0 stays 0 and `packed` stays 0, so the exact-ShapeId + // arm cannot serve this read: only the prefix proof can. + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let v = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(v, 11.0); + assert_eq!( + packed.load(Ordering::Relaxed), + 0, + "a prefix hit answers from the cache and must not prime" + ); + + // A token that no longer matches the receiver's must fall through to + // the ordinary lookup (which still answers, and primes). + cache[PIC_NAMED_PREFIX_TOKEN] = (TOKEN + 1) as i64; + let v = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(v, 11.0); + assert_ne!( + packed.load(Ordering::Relaxed), + 0, + "a stale prefix token must reach the priming miss handler" + ); + } + + /// A site whose cache slot has not been resolved yet reads as a null + /// cache, and the named-prefix arm must not dereference it. + /// + /// The read is of a key the receiver does NOT own, so the miss handler + /// answers from the prototype chain without priming — which is what keeps + /// this test on the arm it is about. (Codegen always passes the address of + /// a real `@perry_ic_N` global; a genuinely null slot only ever reaches a + /// runtime entry from a test or from the write PIC's poly tail.) + #[test] + fn an_unresolved_cache_slot_is_never_dereferenced() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + let own = scope.root_string_ptr(key_of(b"ic_slow_present")); + obj.with_mut_ptr(|o| { + own.with_const_ptr(|k| crate::object::js_object_set_field_by_name(o, k, 3.0)) + }); + let absent = scope.root_string_ptr(key_of(b"ic_slow_absent")); + let packed = AtomicU64::new(0); + // A never-published slot: `pic_slot_peek` must answer null and the + // prefix arm must skip rather than read word 2 out of nothing. + let mut slot: PicCacheSlot = std::ptr::null_mut(); + let v = obj.with_mut_ptr(|o: *mut ObjectHeader| { + absent.with_const_ptr(|k| js_object_get_field_ic_slow(handle(o), k, &mut slot, &packed)) + }); + assert_eq!(v.to_bits(), crate::value::TAG_UNDEFINED); + assert!(slot.is_null(), "an absent key must not resolve a cache"); + assert_eq!(packed.load(Ordering::Relaxed), 0, "and must not prime"); + } +} diff --git a/crates/perry-runtime/src/typed_feedback/tests.rs b/crates/perry-runtime/src/typed_feedback/tests.rs index c12ed2c0c5..4975aa0193 100644 --- a/crates/perry-runtime/src/typed_feedback/tests.rs +++ b/crates/perry-runtime/src/typed_feedback/tests.rs @@ -3038,3 +3038,148 @@ fn function_source_array_literal_keeps_the_array_index_fast_path_armed() { "the emitted guard must still admit plain arrays to the fast path" ); } + +// --------------------------------------------------------------------------- +// `js_class_field_get_ic` — the class-field GET tower's ONE exit. +// +// The helper predates these tests: it was written for the #5391 path-2 full +// outline, which only fires on oversized modules, so its four arms had no +// direct coverage. Every monomorphic `this.field` read whose inline pre-check +// misses now routes through it, so each arm is pinned here: +// +// guard PASS, boxed slot -> the same value the slot holds; +// guard PASS, raw-f64 slot -> the same double, with the intact bit live; +// nullish receiver -> TypeError, not `undefined` (#7153); +// guard FAIL -> one recorded fallback + the by-name answer. +// +// The pass cases assert the site counters too: a helper that silently took the +// fallback on every call would return the right value and pass a value-only +// test, which is exactly the "gate whose subject never ran" shape. +// --------------------------------------------------------------------------- + +// `guards` is a private submodule and its items are not re-exported through +// `super::*`, so name the helper explicitly. +use crate::typed_feedback::guards::js_class_field_get_ic; + +#[test] +fn class_field_get_ic_reads_the_boxed_slot_on_a_guard_pass() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(7401, TypedFeedbackSiteKind::PropertyGet, "obj.x"); + + let class_id = 0x7EED_7401; + let (obj, _, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); + let payload = crate::string::js_string_from_bytes(b"boxed".as_ptr(), 5); + let stored = crate::JSValue::string_ptr(payload); + crate::object::js_object_set_field(obj, 0, stored); + + let got = js_class_field_get_ic(7401, receiver, class_id, expected_shape_id, key_x, 0, 0); + assert_eq!( + got.to_bits(), + stored.bits(), + "a guard PASS must answer with the slot's own NaN-box, bit for bit" + ); + + let site = &typed_feedback_snapshot().sites[0]; + assert_eq!(site.guard_passes, 1); + assert_eq!(site.guard_failures, 0); + assert_eq!( + site.fallback_calls, 0, + "the fast arm must not record a fallback; a helper that always took the \ + by-name path would return the same value" + ); +} + +#[test] +fn class_field_get_ic_reads_the_raw_f64_slot_on_a_guard_pass() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(7402, TypedFeedbackSiteKind::PropertyGet, "obj.x"); + + let class_id = 0x7EED_7402; + let (obj, _, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); + crate::object::js_object_set_field(obj, 0, crate::JSValue::number(5.25)); + let raw_mask = [0b1u64]; + crate::gc::js_gc_init_typed_shape_layout( + obj as u64, + 1, + raw_mask.as_ptr(), + raw_mask.len() as u32, + std::ptr::null(), + 0, + ); + + let got = js_class_field_get_ic(7402, receiver, class_id, expected_shape_id, key_x, 0, 1); + assert_eq!( + got, 5.25, + "a `require_raw_f64` PASS must answer with the raw double the codegen \ + fast block would have loaded" + ); + + let site = &typed_feedback_snapshot().sites[0]; + assert_eq!(site.guard_passes, 1); + assert_eq!(site.fallback_calls, 0); +} + +#[test] +fn class_field_get_ic_records_the_fallback_and_answers_by_name_on_a_guard_fail() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(7403, TypedFeedbackSiteKind::PropertyGet, "obj.x"); + + let class_id = 0x7EED_7403; + let (obj, original_keys, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); + crate::object::js_object_set_field(obj, 0, crate::JSValue::from_bits(5.0f64.to_bits())); + + // Add a key: the shape moves, so the cached (class id, shape id) pair the + // site was compiled against no longer describes this receiver. + let key_y = crate::string::js_string_from_bytes(b"y".as_ptr(), 1); + crate::object::js_object_set_field_by_name(obj, key_y, 10.0); + assert_ne!( + unsafe { crate::object::object_keys_array(obj) }, + original_keys, + "the fixture must actually transition the shape, or the FAIL arm never runs" + ); + + let got = js_class_field_get_ic(7403, receiver, class_id, expected_shape_id, key_x, 0, 0); + assert_eq!( + got.to_bits(), + 5.0f64.to_bits(), + "the FAIL arm must answer by name, not from the stale cached slot" + ); + + let site = &typed_feedback_snapshot().sites[0]; + assert_eq!(site.guard_passes, 0); + assert_eq!(site.guard_failures, 1); + assert_eq!( + site.fallback_calls, 1, + "the helper records the fallback itself — codegen no longer emits the \ + `js_typed_feedback_record_fallback_call` beside it" + ); +} + +#[test] +fn class_field_get_ic_throws_a_type_error_on_a_nullish_receiver() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(7404, TypedFeedbackSiteKind::PropertyGet, "obj.x"); + + let class_id = 0x7EED_7404; + let (obj, _, key_x, _) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); + + for nullish in [crate::value::TAG_UNDEFINED, crate::value::TAG_NULL] { + let receiver = f64::from_bits(nullish); + let threw = catch_runtime_throw(|| { + js_class_field_get_ic(7404, receiver, class_id, expected_shape_id, key_x, 0, 0); + }); + assert!( + threw, + "a field read on a nullish receiver must throw TypeError, not \ + answer `undefined` through the by-name lookup (#7153); bits={nullish:#x}" + ); + } +} diff --git a/crates/perry-transform/src/prop_cse.rs b/crates/perry-transform/src/prop_cse.rs index 7678c2d65f..3b0089fd9f 100644 --- a/crates/perry-transform/src/prop_cse.rs +++ b/crates/perry-transform/src/prop_cse.rs @@ -15,18 +15,18 @@ //! ``` //! //! Every one of those `n.kind` reads compiles to a **full generic -//! property-get diamond** — `pget.recv_sso` / `pget.check_class_ref` / -//! `pget.recv_ok` / the monomorphic-IC guard ladder / `pic.hit` / `pic.miss` / -//! `pget.recv_merge` — about fifty instructions and six branches per site +//! property-get tower** — `pget.recv_ok` / the monomorphic-IC guard ladder / +//! `pic.hit` / `pic.miss` / the polymorphic ways / `pic.miss.call` / +//! `pget.recv_merge` — about forty instructions and six branches per site //! (`expr/property_get/generic_dispatch.rs`). `gc-handoff/apps/interp.ts`'s //! `evalNode` has **53 such sites**, of which `n.kind` accounts for seven and //! `n.op` for seven more. //! //! LLVM `-O3` cannot remove them. GVN dedupes *instructions*, not congruent //! control-flow regions, and every arm of the diamond that is not the inline -//! fast load ends in an opaque call (`js_object_get_field_ic_miss`, -//! `js_object_get_field_by_name_f64`) whose memory effects license nothing. So -//! the redundancy has to be removed before it becomes a diamond — here. +//! fast load ends in an opaque call (`js_object_get_field_ic_slow`) whose +//! memory effects license nothing. So +//! the redundancy has to be removed before it becomes a tower — here. //! //! # The rewrite //! diff --git a/crates/perry/tests/issue_8774_argument_shape_clones.rs b/crates/perry/tests/issue_8774_argument_shape_clones.rs index 0c172b5f0c..d36b77b8bd 100644 --- a/crates/perry/tests/issue_8774_argument_shape_clones.rs +++ b/crates/perry/tests/issue_8774_argument_shape_clones.rs @@ -285,7 +285,8 @@ fn stable_argument_clones_are_direct_reported_and_moving_gc_safe() { ); assert!( !clone_body.contains("shape_descriptor_by_id") - && !clone_body.contains("js_typed_feedback_class_field_get_guard"), + && !clone_body.contains("js_typed_feedback_class_field_get_guard") + && !clone_body.contains("js_class_field_get_ic"), "{clone} rebuilt a field IC diamond:\n{clone_body}" ); assert!(ir.contains(&format!("call double @{clone}("))); diff --git a/docs/src/internals/gc-rooting-invariant.md b/docs/src/internals/gc-rooting-invariant.md index 78dbcf7ff9..20aab1cb9f 100644 --- a/docs/src/internals/gc-rooting-invariant.md +++ b/docs/src/internals/gc-rooting-invariant.md @@ -275,6 +275,8 @@ helper called from a loop body is in-loop too. `movers` and `UnrootedAlloca`) counts `js_gc_loop_safepoint`, anything in `poll_reaching`, **and** anything in `POLL_CAPABLE_RUNTIME` — the runtime helpers that can re-enter JS, such as `js_object_set_field_by_name`, +`js_object_get_field_ic_slow` (the generic property-GET tower's single slow +exit, and since T1 the only GET symbol most emitted sites carry), `js_object_get_field_ic_miss` and `js_closure_call1`. Those are moving with no poll anywhere near them. @@ -293,7 +295,11 @@ like coverage while doing it. Two rounds of this have now been measured: SET verbatim but lowers every GET to `js_object_get_field_by_name_f64`, `js_object_get_field_ic_miss` or `js_typed_feedback_object_get_field_by_name_f64`, none of which were in the - set. Property sets classified `MOVING: YES`, property gets classified + set. (T1 later collapsed the generic tower's six cold arms into one + `js_object_get_field_ic_slow` call, which is in the set for the same reason + and is now the GET symbol a stale-register window is most likely to span — + a size optimisation is exactly the kind of change that can silently retire + the symbol an audit is keyed on.) Property sets classified `MOVING: YES`, property gets classified `MOVING: no`, and 31 `--stale-registers` hits on the gate corpus were dropped by `--moving-only` as a result — including the shape that faults deterministically under `PERRY_GC_PROTECT_FROMSPACE=1 diff --git a/scripts/gc_root_dominance_check.py b/scripts/gc_root_dominance_check.py index d84642e652..015a5f9590 100755 --- a/scripts/gc_root_dominance_check.py +++ b/scripts/gc_root_dominance_check.py @@ -1261,6 +1261,16 @@ def is_collecting(callee): # Same getter/Proxy-capable implementation as the retained three-arg ABI; # the fourth argument only adds a numeric packed cache publication. "js_object_get_field_ic_miss_packed", + # T1: the two slow exits of the inline generic-get tower, and now the ONLY + # property-GET symbols most emitted sites carry. `_slow` reaches + # `get_field_ic_miss_impl` (hence `js_object_get_field_by_name`, hence a + # user getter and `js_proxy_get`); `_nonptr` reaches + # `js_object_get_field_by_name_f64` on every arm. A register held across + # either is exactly as stale as one held across the entries they replaced. + # Omitting them would silently re-open the #7154 GET hole that this set + # exists to close, because the calls they replaced no longer appear at + # those sites. + "js_object_get_field_ic_slow", "js_object_get_field_ic_nonptr", "js_typed_feedback_object_get_field_by_name_f64", "js_array_sort_default", "js_array_sort_with_comparator", "js_array_map", "js_array_filter", "js_typed_array_for_each", @@ -5057,6 +5067,8 @@ def self_test(): _pget_family = {"js_object_get_field_by_name_f64", "js_object_get_field_ic_miss", "js_object_get_field_ic_miss_packed", + "js_object_get_field_ic_slow", + "js_object_get_field_ic_nonptr", "js_typed_feedback_object_get_field_by_name_f64"} if not _pget_family <= POLL_CAPABLE_RUNTIME: print("self-test FAIL: the emitted property-GET dispatch " @@ -5165,27 +5177,36 @@ def self_test(): "code shape rather than on the staleness.", file=sys.stderr) ok = False - # The packed entry is the name emitted by generic property reads now. - # Keeping only the old ABI above made --moving-only silently discard - # the same stale-slot hazard even though both call the same getter. - for label, source, want in ( - ("stale", _SELFTEST_PROPERTY_GET_WINDOW, 1), - ("reloaded", _SELFTEST_PROPERTY_GET_RELOADED, 0)): - packed = os.path.join(td, f"property_get_packed_{label}.ll") - source = source.replace( - "@js_object_get_field_ic_miss(", - "@js_object_get_field_ic_miss_packed(").replace( - "ptr @perry_ic_1)", - "ptr @perry_ic_1, ptr @perry_ic_packed_1)") - with open(packed, "w") as fh: - fh.write(source) - for moving_only in (False, True): - found = _stale_kinds_probe(packed, moving_only=moving_only) - if found.get("slotload", 0) != want: - print(f"self-test FAIL: packed GET {label}, " - f"moving_only={moving_only}: expected {want} " - f"slotload hazards, got {found!r}", file=sys.stderr) - ok = False + # The packed entry is the name emitted by generic property reads now, + # and since T1 the SLOW entry is the only one most sites carry at all: + # the tower's six other calls collapsed into it, so a set that named + # only the older ABIs would classify almost every real property-GET + # window `MOVING: no` — the exact #7154 hole, reopened by a size + # optimisation rather than by a misspelling. Both are asserted in both + # directions against the same fixture. + for entry, extra_args in ( + ("js_object_get_field_ic_miss_packed", + "ptr @perry_ic_1, ptr @perry_ic_packed_1)"), + ("js_object_get_field_ic_slow", + "ptr @perry_ic_1, ptr @perry_ic_packed_1)"), + ("js_object_get_field_ic_nonptr", "i64 4242)")): + for label, source, want in ( + ("stale", _SELFTEST_PROPERTY_GET_WINDOW, 1), + ("reloaded", _SELFTEST_PROPERTY_GET_RELOADED, 0)): + packed = os.path.join(td, f"property_get_{entry}_{label}.ll") + source = source.replace( + "@js_object_get_field_ic_miss(", + "@%s(" % entry).replace( + "ptr @perry_ic_1)", extra_args) + with open(packed, "w") as fh: + fh.write(source) + for moving_only in (False, True): + found = _stale_kinds_probe(packed, moving_only=moving_only) + if found.get("slotload", 0) != want: + print(f"self-test FAIL: {entry} GET {label}, " + f"moving_only={moving_only}: expected {want} " + f"slotload hazards, got {found!r}", file=sys.stderr) + ok = False if (not is_collecting("js_string_compare_value") or is_collecting("js_string_compare")): print("self-test FAIL: only the raw primitive string comparison " diff --git a/scripts/shape_descriptor_census.py b/scripts/shape_descriptor_census.py index 0626294170..18b85fd2e4 100644 --- a/scripts/shape_descriptor_census.py +++ b/scripts/shape_descriptor_census.py @@ -652,7 +652,13 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: "emit_proven_shape_recheck", "emit_class_field_inline_precheck", )), - (raw_element_guard, ("emit_element_shape_field_load",)), + # #10185: the element clone's residual moved out of + # `emit_element_shape_field_load` into the deref helper both the + # per-read path and the shared once-per-iteration prologue call, so the + # ShapeId read this asserts now lives there. Naming the function that + # actually emits the check is what keeps the assertion from going + # vacuous the next time the emitter is split. + (raw_element_guard, ("emit_element_deref_with_residual",)), ): for name in names: body = function_body(source, name) @@ -690,7 +696,7 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: for fragment in ( 'letpacked_present=ctx.block().icmp_ne(I64,&packed_word,"0");', 'letis_plain_object=ctx.block().and(I1,&is_plain_kind,&packed_present);', - 'cond_br(&is_plain_object,&tok_label,&desc_classify_label)', + 'cond_br(&is_plain_object,&tok_label,&cold_label)', 'letpacked_stamp=ctx.block().trunc(I64,&packed_word,I32);', 'lettoken_eq=ctx.block().icmp_eq(I32,&pcid,&packed_stamp);', 'cond_br(&token_eq,&hit_label,&token_miss_label)', diff --git a/test-files/test_gap_generic_get_one_exit_arms.ts b/test-files/test_gap_generic_get_one_exit_arms.ts new file mode 100644 index 0000000000..9c831842bd --- /dev/null +++ b/test-files/test_gap_generic_get_one_exit_arms.ts @@ -0,0 +1,131 @@ +// Every arm of the generic property-get tower, against node's own output. +// +// `lower_generic_property_get` used to expand each of these as its own inline +// block group with its own runtime call: the SSO receiver, the heap-string +// receiver, the INT32 class ref, the nullish TypeError, the non-object +// receiver, the native Map/Set `.size`, the overflow slot, the deleted slot, +// and the Array-subclass named-prefix proof. They now live behind two runtime +// entries (`js_object_get_field_ic_nonptr` for a receiver that is not a heap +// pointer, `js_object_get_field_ic_slow` for one that is), so a defect in any +// of them is no longer a defect in emitted code that some other test might +// notice — it is a defect in one shared function, and this is the witness that +// each arm still answers what node answers. +// +// Every read below goes through `read()`, whose parameter is `any`: that is +// what denies the front end a type proof and lands the access in the generic +// tower rather than in one of the specialised lowerings. + +function read(o: any, k: string): any { + // One generic `obj.prop` site per arm. + if (k === "a") return o.a; + if (k === "length") return o.length; + if (k === "size") return o.size; + if (k === "kind") return o.kind; + if (k === "charCodeAt") return o.charCodeAt; + return o.zzz; +} + +// --- receiver tags that are not heap pointers ------------------------------- + +// SSO string (short enough to live inside the NaN box) and a heap string. +const sso: any = "hi"; +const heap: any = "a considerably longer string"; +console.log("sso.length", read(sso, "length")); +console.log("heap.length", read(heap, "length")); +console.log("sso.charCodeAt is fn", typeof read(sso, "charCodeAt")); +console.log("heap.zzz", read(heap, "zzz")); + +// A class object read as a value: its static field, not an instance field. +class C { + static kind = "static-c"; + x = 1; +} +const cref: any = C; +console.log("C.kind", read(cref, "kind")); + +// Non-nullish primitives: no auto-boxing, no throw, just `undefined`. +console.log("3.5.zzz", read(3.5 as any, "zzz")); +console.log("true.zzz", read(true as any, "zzz")); + +// --- heap receivers that are not ordinary objects --------------------------- + +const m: any = new Map(); +m.set(1, 2); +const st: any = new Set(); +st.add(9); +console.log("map.size", read(m, "size"), "set.size", read(st, "size")); + +const arr: any = [1, 2, 3]; +console.log("arr.length", read(arr, "length")); + +// --- ordinary objects: hit, hole, re-add ------------------------------------ + +const o: any = { a: 1, b: 2 }; +console.log("o.a", read(o, "a")); +delete o.a; +console.log("o.a deleted", read(o, "a")); +o.a = 7; +console.log("o.a again", read(o, "a")); + +// A field past the inline region (the overflow slot), read repeatedly so the +// site primes and then takes the primed path, then tombstoned. +const big: any = {}; +big.p0 = 0; +big.p1 = 1; +big.p2 = 2; +big.p3 = 3; +big.p4 = 4; +for (let i = 0; i < 3; i++) console.log("big.p4", big.p4); +delete big.p4; +console.log("big.p4 deleted", big.p4); + +// Absent own key: the prototype chain answers. +const proto = { inherited: "yes" }; +const child: any = Object.create(proto); +console.log("child.inherited", child.inherited); + +// A descriptor-bearing receiver: the accessor must fire on EVERY read, which +// is what forbids a raw-slot hit from ever being primed for it. +let calls = 0; +const acc: any = {}; +Object.defineProperty(acc, "g", { + get() { + calls += 1; + return calls; + }, +}); +console.log("acc.g", acc.g, acc.g, acc.g, "calls", calls); + +// --- nullish receivers throw a node-shaped TypeError ------------------------ + +try { + const n: any = null; + console.log(n.foo); +} catch (e: any) { + console.log("caught null:", e.message); +} +try { + const u: any = undefined; + console.log(u.bar); +} catch (e: any) { + console.log("caught undefined:", e.message); +} + +// --- shape rotation: the bounded polymorphic ways --------------------------- + +class S1 { + x = 1; +} +class S2 { + a = 0; + x = 2; +} +class S3 { + a = 0; + b = 0; + x = 3; +} +const rot: any[] = [new S1(), new S2(), new S3()]; +let sum = 0; +for (let i = 0; i < 300; i++) sum += rot[i % rot.length].x; +console.log("poly sum", sum); diff --git a/test-files/test_gap_json_record_loop_clone.ts b/test-files/test_gap_json_record_loop_clone.ts index 0a60a97359..363fb7a04b 100644 --- a/test-files/test_gap_json_record_loop_clone.ts +++ b/test-files/test_gap_json_record_loop_clone.ts @@ -270,3 +270,301 @@ function sumTwoFields(rows: any, count: number): number { return sum; } console.log("two-fields:", sumTwoFields(moduleRows, moduleLength)); + +// --------------------------------------------------------------------------- +// 9. #10185: the LOOP-CARRIED index (`random`), the K-statement accumulator +// fold, and the two non-numeric reads. +// +// Every case below is a MISCOMPILE if it is wrong, not a slow path: +// +// * the carried write-back is placed at the END of the iteration, so a +// mid-iteration side exit leaves the real binding holding that +// iteration's ENTRY value and the slow clone advances the recurrence +// exactly once. A write-back at the update site double-applies it, and +// every later index is silently a different (still valid) record; +// * K accumulator statements fold to one store for the same reason: a +// side exit from the third read must not leave the first two applied; +// * `.length` and the ternary read a NaN-boxed word and assume a +// representation, so each tag-tests it and side-exits otherwise. +// --------------------------------------------------------------------------- + +// The benchmark's `random` mode, with the `const index = cursor` alias. +function randomSum(rows: any, count: number, n: number): number { + let sum = 0; + let cursor = 0; + for (let i = 0; i < count; i++) { + cursor = (cursor * 17 + 7) % n; + const index = cursor; + sum += rows[index].id; + } + return sum; +} + +// The same recurrence subscripted DIRECTLY, with the carried value read after +// the loop — the clone owes a write-back, and this is what observes it. +function randomSumAndCursor( + rows: any, + count: number, + n: number, + start: number, +): string { + let sum: any = 0; + let cursor = start; + for (let i = 0; i < count; i++) { + cursor = (cursor * 17 + 7) % n; + sum += rows[cursor].id; + } + return String(sum) + "|" + String(cursor); +} + +// A multiplier large enough that `a * cursor` leaves the range an `f64` +// represents exactly. JavaScript evaluates this in doubles, so an i64 +// recurrence would NOT agree — the matcher must decline and let the ordinary +// lowering run. +function randomSumWideMultiplier( + rows: any, + count: number, + n: number, +): string { + let sum = 0; + let cursor = 3; + for (let i = 0; i < count; i++) { + cursor = (cursor * 600000000 + 7) % n; + sum += rows[cursor].id; + } + return String(sum) + "|" + String(cursor); +} + +// A recurrence with an even step, so a fractional carried value still lands +// on whole-number indices after the first update. +function randomSumEvenStep( + rows: any, + count: number, + n: number, + start: number, +): string { + let sum = 0; + let cursor = start; + for (let i = 0; i < count; i++) { + cursor = (cursor * 2 + 7) % n; + sum += rows[cursor].id; + } + return String(sum) + "|" + String(cursor); +} + +// The benchmark's `fields` mode: three accumulator statements over ONE +// element, one through a string and one through a boolean. +function fieldsSum(rows: any, count: number, n: number): number { + let sum = 0; + for (let i = 0; i < count; i++) { + const index = i % n; + sum += rows[index].id; + sum += rows[index].name.length; + sum += rows[index].active ? 1 : 0; + } + return sum; +} + +const randomRows: any = JSON.parse(buildRecords(40, "pad")); +console.log("random-sum:", randomSum(randomRows, 200, randomRows.length)); +console.log( + "random-sum-again:", + randomSum(randomRows, 200, randomRows.length), +); +console.log( + "random-cursor-live-after:", + randomSumAndCursor(randomRows, 200, randomRows.length, 0), +); +// An odd trip count observes a different point of the cycle than the run +// above, so the write-back is checked at two places rather than one. +console.log( + "random-cursor-live-after-odd:", + randomSumAndCursor(randomRows, 201, randomRows.length, 0), +); +// A modulus smaller than the array. 13 is chosen because `(17c + 7) % 13` +// walks six distinct indices from 0 — several plausible moduli make the +// recurrence a fixed point at 0, where every assertion below would hold of a +// loop that read one element forever. +console.log( + "random-small-modulus:", + randomSumAndCursor(randomRows, 50, 13, 0), +); +console.log( + "random-modulus-one:", + randomSumAndCursor(randomRows, 9, 1, 0), +); + +// Entry values the preheader must reject: JS `%` returns a NEGATIVE remainder +// for a negative dividend, and a fractional index is not an index at all. +// Both take the slow clone and keep ordinary JavaScript semantics. +try { + console.log( + "random-negative-start:", + randomSumAndCursor(randomRows, 4, 5, -3), + ); +} catch (err) { + console.log("random-negative-start-threw:", String(err).slice(0, 9)); +} +// A FRACTIONAL entry value. The step is even so every index the loop +// actually reads is a whole number — the divergence being checked is the +// preheader's, not the subscript's: if the clone materialized `0.5` as the +// i32 `0` it would start the recurrence one step early and every index would +// shift, which the sum and the final cursor both show. +console.log( + "random-fractional-start:", + randomSumEvenStep(randomRows, 6, 5, 0.5), +); +console.log( + "random-integral-start-same-step:", + randomSumEvenStep(randomRows, 6, 5, 0), +); +// `x % 0` is NaN, and `rows[NaN]` is `undefined`. +try { + console.log("random-modulus-zero:", randomSumAndCursor(randomRows, 3, 0, 0)); +} catch (err) { + console.log("random-modulus-zero-threw:", String(err).slice(0, 9)); +} +// A modulus past the array's length runs off the end. +try { + console.log( + "random-modulus-past-length:", + randomSumAndCursor(randomRows, 30, 500, 0), + ); +} catch (err) { + console.log("random-modulus-past-length-threw:", String(err).slice(0, 9)); +} +console.log( + "random-wide-multiplier:", + randomSumWideMultiplier(randomRows, 25, 11), +); + +// THE write-back test. The array is homogeneous in SHAPE, so the clone is +// entered — but one `id` is a string, so the per-read Number tag test side +// -exits mid-iteration and the slow clone re-runs it. If the recurrence had +// already been committed, the slow clone advances it a SECOND time and both +// the sum and the final cursor drift. +// `(17c + 7) % 5` walks 0 -> 2 -> 1 -> 4 -> 0, so index 2 — the string one — +// really is read. A modulus that skipped it would make this whole case +// vacuous while still printing a matching number. +const randomMixed: any = JSON.parse( + '[{"id":0},{"id":1},{"id":"two"},{"id":3},{"id":4}]', +); +console.log( + "random-side-exit:", + randomSumAndCursor(randomMixed, 30, randomMixed.length, 0), +); +console.log( + "random-side-exit-again:", + randomSumAndCursor(randomMixed, 30, randomMixed.length, 0), +); + +// The recurrence over a HETEROGENEOUS array declines at the array level. +console.log( + "random-hetero:", + randomSumAndCursor(hetero, 16, hetero.length, 0), +); + +// --------------------------------------------------------------------------- +// 9b. The `fields` shape. +// --------------------------------------------------------------------------- + +// SSO names (<= 5 bytes, packed into the NaN-box) and heap names in the SAME +// array — the shape is identical either way, and the `.length` decode has to +// handle both. +const mixedNames: any = JSON.parse( + '[{"id":1,"name":"ab","active":true},' + + '{"id":2,"name":"user_222","active":false},' + + '{"id":3,"name":"","active":true},' + + '{"id":4,"name":"abcde","active":false},' + + '{"id":5,"name":"abcdef","active":true}]', +); +console.log("fields-sso-and-heap:", fieldsSum(mixedNames, 25, mixedNames.length)); +console.log( + "fields-sso-and-heap-again:", + fieldsSum(mixedNames, 25, mixedNames.length), +); + +// A NON-STRING `name`: `.length` of a number is `undefined`, so the sum goes +// NaN — and the clone must side-exit rather than decode a double as a string +// header. The fold is what keeps the earlier `id` from being applied twice. +const numberName: any = JSON.parse( + '[{"id":1,"name":"ab","active":true},{"id":2,"name":7,"active":true},' + + '{"id":3,"name":"cd","active":false}]', +); +console.log("fields-number-name:", fieldsSum(numberName, 9, numberName.length)); + +// A NULL `name`: `null.length` throws, exactly as JS says. +const nullName: any = JSON.parse( + '[{"id":1,"name":"ab","active":true},{"id":2,"name":null,"active":true}]', +); +try { + console.log("fields-null-name:", fieldsSum(nullName, 4, nullName.length)); +} catch (err) { + console.log("fields-null-name-threw:", String(err).slice(0, 9)); +} + +// NON-BOOLEAN `active` values. JS truthiness of `0`, `""`, `"no"` and `null` +// is a runtime question the clone is not allowed to guess: only the two +// boolean singletons are admitted and everything else side-exits. +const truthyActive: any = JSON.parse( + '[{"id":1,"name":"ab","active":true},{"id":2,"name":"cd","active":0},' + + '{"id":3,"name":"ef","active":"no"},{"id":4,"name":"gh","active":""},' + + '{"id":5,"name":"ij","active":null},{"id":6,"name":"kl","active":false}]', +); +console.log( + "fields-truthiness:", + fieldsSum(truthyActive, 24, truthyActive.length), +); +console.log( + "fields-truthiness-again:", + fieldsSum(truthyActive, 24, truthyActive.length), +); + +// Two accumulator statements where the SECOND read side-exits: the first must +// not be applied twice when the slow clone re-runs the iteration. +function twoStatementSum(rows: any, count: number, n: number): number { + let sum = 0; + for (let i = 0; i < count; i++) { + const index = i % n; + sum += rows[index].id; + sum += rows[index].score; + } + return sum; +} +const secondFieldMixed: any = JSON.parse( + '[{"id":1,"score":10},{"id":2,"score":"x"},{"id":3,"score":30}]', +); +console.log( + "fold-side-exit:", + twoStatementSum(secondFieldMixed, 9, secondFieldMixed.length), +); + +// Heterogeneous shapes with the multi-read body. +console.log("fields-hetero:", fieldsSum(hetero, 12, hetero.length)); + +// --------------------------------------------------------------------------- +// 9c. Property names the CLASS-keyed arm denies. A parsed record's own data +// property SHADOWS the prototype name it collides with, and the +// shape-keyed preheader resolves it from the runtime shape table, so +// `name` and `length` are ordinary inline slots here. `name` in +// particular is what the access benchmark's own `fields` mode reads. +// --------------------------------------------------------------------------- +function ownNamedFields(rows: any, count: number, n: number): number { + let sum = 0; + for (let i = 0; i < count; i++) { + const index = i % n; + sum += rows[index].length; + sum += rows[index].size; + sum += rows[index].name.length; + } + return sum; +} +const ownNames: any = JSON.parse( + '[{"length":3,"size":4,"name":"ab"},{"length":5,"size":6,"name":"cdef"},' + + '{"length":7,"size":8,"name":"ghijklm"}]', +); +console.log("own-named-fields:", ownNamedFields(ownNames, 12, ownNames.length)); +console.log( + "own-named-fields-again:", + ownNamedFields(ownNames, 12, ownNames.length), +);