From 66294fb7ab5d7a93f99b6182dc49607cad6af6a7 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Sat, 25 Jul 2026 12:28:10 -0400 Subject: [PATCH] =?UTF-8?q?perf:=20narrow=20the=20byte=E2=86=92UTF-16=20ma?= =?UTF-8?q?p=20to=20u8=20deltas=20filled=20by=20run-memset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/tsv_lang/CLAUDE.md | 2 +- crates/tsv_lang/src/location.rs | 644 ++++++++++++++++++++++++-------- docs/performance.md | 9 + 3 files changed, 501 insertions(+), 154 deletions(-) diff --git a/crates/tsv_lang/CLAUDE.md b/crates/tsv_lang/CLAUDE.md index ea5ac097b..5cee282fe 100644 --- a/crates/tsv_lang/CLAUDE.md +++ b/crates/tsv_lang/CLAUDE.md @@ -9,7 +9,7 @@ All language crates (tsv_ts, tsv_css, tsv_svelte) depend on tsv_lang. It provide Each module's visibility (in parens) reflects `pub use`-only modules (private) vs directly-imported modules (`pub mod`, used as `tsv_lang::doc::{...}` etc.). - `span` (`span.rs`, private) — `Span { start: u32, end: u32 }` — compact source positions -- `location` (`location.rs`, private) — `LocationTracker` (line/column via binary search on line starts, fronted by a 1-entry line-range cache that turns the sequential-emission common case into an O(1) range check), `ByteToCharMap` (byte → UTF-16 code-unit offsets; `identity()` for byte-space passthrough), and `LocationMapper` (tracker + map bundle the AST-conversion layers thread — with a real map it emits final char-space positions during conversion, fusing out the post-conversion translation walk; with the identity map it is exact byte-space passthrough). The `no-locations` emission path skips the line-start scan entirely — it builds a line-data-free tracker via `LocationTracker::new_map_only` (stub `line_starts`, byte→UTF-16 map only) and emits `start`/`end` offsets with no line/column +- `location` (`location.rs`, private) — `LocationTracker` (line/column via binary search on line starts, fronted by a 1-entry line-range cache that turns the sequential-emission common case into an O(1) range check), `ByteToCharMap` (byte → UTF-16 code-unit offsets, stored as a dense `byte − utf16` **delta** table narrowed to `u8` unless the source's multibyte characters are dense enough to outgrow it — lookup stays one O(1) load, `pos = byte − delta[byte]`; `identity()` for byte-space passthrough, and for an ASCII-only source, which carries no table at all), and `LocationMapper` (tracker + map bundle the AST-conversion layers thread — with a real map it emits final char-space positions during conversion, fusing out the post-conversion translation walk; with the identity map it is exact byte-space passthrough). The `no-locations` emission path skips the line-start scan entirely — it builds a line-data-free tracker via `LocationTracker::new_map_only` (stub `line_starts`, byte→UTF-16 map only) and emits `start`/`end` offsets with no line/column - `error` (`error.rs`, private) — `ParseError` with context extraction and caret formatting - `config` (`config.rs`, private) — `PRINT_WIDTH` / `TAB_WIDTH` / `INDENT` consts + `EmbedContext` / `LayoutMode` (no runtime config) - `doc` (`doc/*.rs`, pub) — Document builder — arena-based Prettier-compatible IR diff --git a/crates/tsv_lang/src/location.rs b/crates/tsv_lang/src/location.rs index 052fa0512..5795fca5a 100644 --- a/crates/tsv_lang/src/location.rs +++ b/crates/tsv_lang/src/location.rs @@ -14,10 +14,21 @@ pub struct Position { /// /// Rust strings are byte-indexed, but JS (and Svelte/acorn) uses UTF-16 /// code unit indices. For ASCII-only sources, byte == char offset, so the map is empty. -/// For sources with multibyte UTF-8 characters, the map stores the UTF-16 code unit -/// offset for each byte position. (A sparse per-multibyte-char representation -/// with binary-search lookup was measured at +3% instructions on a -/// multibyte-dense corpus — the O(1) dense lookup wins; don't re-derive.) +/// For sources with multibyte UTF-8 characters, the map stores, for each byte +/// position, the **delta** `byte − utf16` rather than the absolute UTF-16 offset: +/// the delta is what a multibyte character actually contributes, it is +/// non-decreasing across the source, and it stays small enough to hold in a +/// `u8` on any source whose multibyte characters are sparse — which is the +/// common case (a real TS corpus runs ~1 non-ASCII character per 1000 bytes, +/// yet ~90% of its files contain at least one). Narrowing the table from `u32` +/// to `u8` cuts its construction write traffic and its cache footprint 4×; a +/// source dense enough to outgrow `u8` widens back to `u32`. +/// +/// Lookup stays a single O(1) indexed load either way (`pos = byte − +/// delta[byte]`). (A sparse per-multibyte-char representation with +/// binary-search *lookup* was measured at +3% instructions on a +/// multibyte-dense corpus — the O(1) dense lookup wins; don't re-derive. That +/// result is about lookup only, and the delta narrowing leaves it intact.) /// /// Characters in the BMP (U+0000-U+FFFF) count as 1 UTF-16 code unit. /// Characters outside the BMP (U+10000+, e.g., most emoji) count as 2 (surrogate pair). @@ -27,10 +38,17 @@ pub struct Position { /// position inside a multibyte character resolves to that character's start. #[derive(Debug)] pub struct ByteToCharMap { - /// For each byte position, the corresponding UTF-16 code unit offset. - /// Empty for ASCII-only sources (byte == char offset). - offsets: Vec, - has_multibyte: bool, + deltas: Deltas, +} + +/// The `byte − utf16` delta table, one entry per source byte plus an +/// end-of-source sentinel. `Identity` carries no table at all (ASCII-only +/// source, or the byte-space passthrough mode). +#[derive(Debug)] +enum Deltas { + Identity, + Narrow(Vec), + Wide(Vec), } impl ByteToCharMap { @@ -41,25 +59,8 @@ impl ByteToCharMap { if source.is_ascii() { return Self::identity(); } - - let mut offsets = vec![0u32; source.len() + 1]; - let mut utf16_idx = 0u32; - for (byte_idx, ch) in source.char_indices() { - // Every byte of the character gets the character's UTF-16 offset, - // so a byte position inside a multibyte char resolves to that - // char's start. - for offset in &mut offsets[byte_idx..byte_idx + ch.len_utf8()] { - *offset = utf16_idx; - } - // Characters outside BMP need 2 UTF-16 code units (surrogate pair) - utf16_idx += ch.len_utf16() as u32; - } - offsets[source.len()] = utf16_idx; - - Self { - offsets, - has_multibyte: true, - } + let mut no_lines = Vec::new(); + build_map(source, &mut no_lines, LineRule::None) } /// The identity map: every byte offset translates to itself. @@ -70,30 +71,249 @@ impl ByteToCharMap { /// line up; the final fused emit uses the real map). pub const fn identity() -> Self { Self { - offsets: Vec::new(), - has_multibyte: false, + deltas: Deltas::Identity, } } /// Convert a byte offset to a UTF-16 code unit offset /// /// For ASCII-only sources, returns the byte offset unchanged. Offsets - /// past the end of the source also translate to themselves. + /// past the end of the source also translate to themselves (a missing + /// entry is a zero delta). #[inline] pub fn byte_to_char(&self, byte_offset: u32) -> u32 { - if !self.has_multibyte { - return byte_offset; + match &self.deltas { + Deltas::Identity => byte_offset, + Deltas::Narrow(deltas) => { + byte_offset - u32::from(deltas.get(byte_offset as usize).copied().unwrap_or(0)) + } + Deltas::Wide(deltas) => { + byte_offset - deltas.get(byte_offset as usize).copied().unwrap_or(0) + } } - self.offsets - .get(byte_offset as usize) - .copied() - .unwrap_or(byte_offset) } /// Whether the source contains multibyte UTF-8 characters #[inline] pub fn has_multibyte(&self) -> bool { - self.has_multibyte + !matches!(self.deltas, Deltas::Identity) + } +} + +/// Which line-terminator rule the fused map builder applies. `None` skips line +/// starts entirely — the map-only path. +/// +/// A runtime parameter rather than a `const` one: it is consulted once per +/// *run* (the stretch between two multibyte characters), never per byte, so +/// specializing on it buys nothing measurable and costs three extra +/// monomorphizations of the builder in every shipped artifact. +#[derive(Clone, Copy, PartialEq, Eq)] +enum LineRule { + None, + Lf, + Ecmascript, +} + +/// One delta-table element width. `u8` is what a sparse-multibyte source needs; +/// `u32` is the widening fallback for a source whose deltas outgrow it. +trait DeltaElem: Copy { + /// The largest delta this element can hold. The `u32` arm's check folds + /// away — a delta never exceeds the source length. + const MAX_DELTA: u32; + fn from_delta(delta: u32) -> Self; + fn into_deltas(deltas: Vec) -> Deltas; +} + +impl DeltaElem for u8 { + const MAX_DELTA: u32 = Self::MAX as u32; + #[inline] + fn from_delta(delta: u32) -> Self { + delta as Self + } + fn into_deltas(deltas: Vec) -> Deltas { + Deltas::Narrow(deltas) + } +} + +impl DeltaElem for u32 { + const MAX_DELTA: u32 = Self::MAX; + #[inline] + fn from_delta(delta: u32) -> Self { + delta + } + fn into_deltas(deltas: Vec) -> Deltas { + Deltas::Wide(deltas) + } +} + +/// Where a narrow build stopped because the next character's deltas would not +/// fit — the resume point for the wide continuation. +struct Outgrown { + /// Byte index of the character that didn't fit; also the number of table + /// entries already written, since the scan writes exactly one per byte. + at: usize, + /// The running delta as of `at`. + delta: u32, +} + +/// Index of the first non-ASCII byte at or after `from`, or `bytes.len()`. +/// +/// Word-at-a-time: the multibyte characters this splits the source on are +/// sparse, so the scan between them is the bulk of construction. +#[inline] +fn next_non_ascii(bytes: &[u8], from: usize) -> usize { + const HIGH_BITS: u64 = 0x8080_8080_8080_8080; + let mut i = from; + while let Some(chunk) = bytes[i..].first_chunk::<8>() { + let hits = u64::from_le_bytes(*chunk) & HIGH_BITS; + if hits != 0 { + // Little-endian: the lowest set bit is the high bit of the + // earliest non-ASCII byte in the word. + return i + (hits.trailing_zeros() / 8) as usize; + } + i += 8; + } + while i < bytes.len() && bytes[i] < 0x80 { + i += 1; + } + i +} + +/// Fill the `byte − utf16` delta table (and, per `line_rule`, the line starts), +/// resuming at byte `from` with running delta `delta`. +/// +/// The scan walks *runs*: everything between two multibyte characters is a +/// constant delta (one fill) and pure ASCII (so the line scan is the same +/// byte-level one the all-ASCII fast path uses, shared verbatim rather than +/// re-derived). Only the multibyte characters themselves are handled per byte. +/// +/// Returns `Err(Outgrown)` — leaving `deltas` and `line_starts` correct and +/// complete up to that point, for the wide continuation to pick up — if the +/// next character's deltas would not fit `T`. +fn build_deltas( + source: &str, + deltas: &mut Vec, + line_starts: &mut Vec, + from: usize, + mut delta: u32, + line_rule: LineRule, +) -> Result<(), Outgrown> { + let bytes = source.as_bytes(); + let mut i = from; + + while i < bytes.len() { + let run_end = next_non_ascii(bytes, i); + if run_end > i { + match line_rule { + LineRule::Lf => ascii_lf_line_starts_into(&bytes[i..run_end], i, line_starts), + LineRule::Ecmascript => { + ascii_ecmascript_line_starts_into(&bytes[i..run_end], i, line_starts); + } + LineRule::None => {} + } + // The whole ASCII run shares the running delta. + deltas.resize(run_end, T::from_delta(delta)); + i = run_end; + if i == bytes.len() { + break; + } + } + + // A multibyte character starts at `i`. Each of its bytes gets the + // delta that resolves it to the character's own start, so a byte + // position inside the character reads back as that start. + let lead = bytes[i]; + let len = utf8_len(lead); + // The character's last byte carries `delta + len - 1`, the widest value + // it writes — and the delta it leaves behind is no wider than that. + if T::MAX_DELTA - delta < len as u32 - 1 { + return Err(Outgrown { at: i, delta }); + } + match len { + 2 => { + deltas.push(T::from_delta(delta)); + deltas.push(T::from_delta(delta + 1)); + delta += 1; + i += 2; + } + 3 => { + deltas.push(T::from_delta(delta)); + deltas.push(T::from_delta(delta + 1)); + deltas.push(T::from_delta(delta + 2)); + delta += 2; + // U+2028 / U+2029 are line terminators under the ECMAScript + // rule only, and are the sole multibyte ones (E2 80 A8/A9). + if line_rule == LineRule::Ecmascript + && lead == 0xE2 + && bytes[i + 1] == 0x80 + && matches!(bytes[i + 2], 0xA8 | 0xA9) + { + line_starts.push(i + 3); + } + i += 3; + } + _ => { + // Astral: 4 bytes, 2 UTF-16 code units (surrogate pair). + deltas.push(T::from_delta(delta)); + deltas.push(T::from_delta(delta + 1)); + deltas.push(T::from_delta(delta + 2)); + deltas.push(T::from_delta(delta + 3)); + delta += 2; + i += 4; + } + } + } + + // The trailing run plus the end-of-source sentinel. + deltas.resize(bytes.len() + 1, T::from_delta(delta)); + Ok(()) +} + +/// UTF-8 length from a lead byte (only ever called on one, so the 2-byte arm +/// covers the whole `0x80..0xE0` range). +#[inline] +fn utf8_len(lead: u8) -> usize { + if lead < 0xE0 { + 2 + } else if lead < 0xF0 { + 3 + } else { + 4 + } +} + +/// Build the delta table for a source known to contain a multibyte character, +/// narrowing to `u8` when the source's deltas fit — which is every source whose +/// multibyte characters are sparse. `line_starts` must already hold its leading +/// `0` unless `line_rule` is `None`. +fn build_map(source: &str, line_starts: &mut Vec, line_rule: LineRule) -> ByteToCharMap { + let mut narrow: Vec = Vec::with_capacity(source.len() + 1); + let Err(outgrown) = build_deltas::(source, &mut narrow, line_starts, 0, 0, line_rule) + else { + return ByteToCharMap { + deltas: u8::into_deltas(narrow), + }; + }; + + // A multibyte-dense source (hundreds of non-ASCII characters) whose running + // delta outgrew `u8`. Widen what the narrow scan already produced and + // continue from where it stopped — the source is scanned once either way. + let mut wide: Vec = Vec::with_capacity(source.len() + 1); + wide.extend(narrow.iter().copied().map(u32::from)); + drop(narrow); + debug_assert_eq!(wide.len(), outgrown.at, "widened prefix must be complete"); + // Infallible: `u32::MAX_DELTA` is unreachable — a delta never exceeds the + // source length, and spans are `u32` throughout. + let _ = build_deltas::( + source, + &mut wide, + line_starts, + outgrown.at, + outgrown.delta, + line_rule, + ); + ByteToCharMap { + deltas: u32::into_deltas(wide), } } @@ -253,44 +473,8 @@ impl LocationTracker { } let mut line_starts = vec![0]; - let mut offsets = vec![0u32; source.len() + 1]; - let mut utf16_idx = 0u32; - let mut chars = source.char_indices().peekable(); - while let Some((i, ch)) = chars.next() { - let len_utf8 = ch.len_utf8(); - // Every byte of the character gets the character's UTF-16 offset, - // so a byte position inside a multibyte char resolves to that - // char's start. - for offset in &mut offsets[i..i + len_utf8] { - *offset = utf16_idx; - } - utf16_idx += ch.len_utf16() as u32; - match ch { - '\n' | '\u{2028}' | '\u{2029}' => line_starts.push(i + len_utf8), - '\r' => { - // CRLF counts as a single line terminator; the consumed - // '\n' still fills its map slot and counts one UTF-16 unit. - if let Some(&(j, '\n')) = chars.peek() { - chars.next(); - offsets[j] = utf16_idx; - utf16_idx += 1; - line_starts.push(j + 1); - } else { - line_starts.push(i + 1); - } - } - _ => {} - } - } - offsets[source.len()] = utf16_idx; - - ( - Self::with_line_starts(line_starts), - ByteToCharMap { - offsets, - has_multibyte: true, - }, - ) + let map = build_map(source, &mut line_starts, LineRule::Ecmascript); + (Self::with_line_starts(line_starts), map) } /// Build the LF-only tracker (Svelte's `locate-character` convention — only @@ -307,29 +491,8 @@ impl LocationTracker { } let mut line_starts = vec![0]; - let mut offsets = vec![0u32; source.len() + 1]; - let mut utf16_idx = 0u32; - for (i, ch) in source.char_indices() { - // Every byte of the character gets the character's UTF-16 offset, - // so a byte position inside a multibyte char resolves to that - // char's start. - for offset in &mut offsets[i..i + ch.len_utf8()] { - *offset = utf16_idx; - } - utf16_idx += ch.len_utf16() as u32; - if ch == '\n' { - line_starts.push(i + 1); - } - } - offsets[source.len()] = utf16_idx; - - ( - Self::with_line_starts(line_starts), - ByteToCharMap { - offsets, - has_multibyte: true, - }, - ) + let map = build_map(source, &mut line_starts, LineRule::Lf); + (Self::with_line_starts(line_starts), map) } /// A line-data-free tracker: only the byte→char `map` half of a @@ -391,11 +554,7 @@ impl LocationTracker { /// convention: only `\n` starts a line — no CR/CRLF fusing). fn ascii_lf_line_starts(bytes: &[u8]) -> Vec { let mut line_starts = vec![0]; - for (i, &b) in bytes.iter().enumerate() { - if b == b'\n' { - line_starts.push(i + 1); - } - } + ascii_lf_line_starts_into(bytes, 0, &mut line_starts); line_starts } @@ -403,28 +562,247 @@ fn ascii_lf_line_starts(bytes: &[u8]) -> Vec { /// possible, so line terminators are single bytes with CRLF fusing. fn ascii_ecmascript_line_starts(bytes: &[u8]) -> Vec { let mut line_starts = vec![0]; + ascii_ecmascript_line_starts_into(bytes, 0, &mut line_starts); + line_starts +} + +/// Append the LF-only line starts of an ASCII run to `line_starts`, offset by +/// the run's `base` position in the source. +/// +/// The multibyte builder splits the source into ASCII runs at its multibyte +/// characters, so each run's line scan is exactly the all-ASCII one — shared +/// with the fast path rather than re-derived. A run boundary can never split a +/// line terminator: every terminator this rule recognizes is one ASCII byte. +fn ascii_lf_line_starts_into(bytes: &[u8], base: usize, line_starts: &mut Vec) { + for (i, &b) in bytes.iter().enumerate() { + if b == b'\n' { + line_starts.push(base + i + 1); + } + } +} + +/// Append the ECMAScript-rule line starts of an ASCII run to `line_starts`, +/// offset by the run's `base` position in the source. +/// +/// A CRLF pair can never straddle a run boundary — a run only ends at a +/// non-ASCII byte, which `\n` is not — so a `\r` at the end of a run is a lone +/// CR, exactly as this scan reads it. +fn ascii_ecmascript_line_starts_into(bytes: &[u8], base: usize, line_starts: &mut Vec) { let mut i = 0; while i < bytes.len() { match bytes[i] { - b'\n' => line_starts.push(i + 1), + b'\n' => line_starts.push(base + i + 1), b'\r' => { // CRLF counts as a single line terminator if bytes.get(i + 1) == Some(&b'\n') { i += 1; } - line_starts.push(i + 1); + line_starts.push(base + i + 1); } _ => {} } i += 1; } - line_starts } #[cfg(test)] mod tests { use super::*; + /// The dense absolute-`u32` table the delta representation replaced, kept + /// as this module's arithmetic oracle. + /// + /// **No corpus can grade this translation.** A byte→UTF-16 error only + /// changes emitted output at a position the wire actually carries, on a + /// file that actually has a multibyte character before it — so a wrong + /// table survives fixture, format-diff, and wire-diff gates. The delta + /// table is therefore graded against this shape *exhaustively* + /// (`test_delta_map_matches_dense_reference_exhaustive`), and by nothing + /// else. + fn reference_offsets(source: &str) -> Vec { + let mut offsets = vec![0u32; source.len() + 1]; + let mut utf16_idx = 0u32; + for (byte_idx, ch) in source.char_indices() { + for offset in &mut offsets[byte_idx..byte_idx + ch.len_utf8()] { + *offset = utf16_idx; + } + utf16_idx += ch.len_utf16() as u32; + } + offsets[source.len()] = utf16_idx; + offsets + } + + /// Every map constructor, at every offset in and past the source. + fn assert_map_matches_reference(source: &str, label: &str) { + // One table per source — a per-offset rebuild is quadratic. + let reference = reference_offsets(source); + let reference_byte_to_char = |byte_offset: u32| -> u32 { + if source.is_ascii() { + return byte_offset; + } + reference + .get(byte_offset as usize) + .copied() + .unwrap_or(byte_offset) + }; + + let plain = ByteToCharMap::new(source); + let (ecma_tracker, ecma_map) = LocationTracker::new_ecmascript_with_map(source); + let (lf_tracker, lf_map) = LocationTracker::new_with_map(source); + let (_, map_only) = LocationTracker::new_map_only(source); + + let maps = [ + (&plain, "new"), + (&ecma_map, "new_ecmascript_with_map"), + (&lf_map, "new_with_map"), + (&map_only, "new_map_only"), + ]; + + for b in 0..=(source.len() as u32 + 2) { + let expected = reference_byte_to_char(b); + for (map, which) in maps { + assert_eq!( + map.byte_to_char(b), + expected, + "{which} diverges at byte {b} on {label} {source:?}" + ); + } + } + + // An ASCII-only source must carry no table at all — the flag every + // `LocationMapper` column branch reads. + for (map, which) in maps { + assert_eq!( + map.has_multibyte(), + !source.is_ascii(), + "{which} has_multibyte wrong on {label} {source:?}" + ); + } + + // The line halves of the fused constructors, against the char-walking + // trackers they must stay byte-identical to. + assert_eq!( + ecma_tracker.line_starts, + LocationTracker::new_ecmascript(source).line_starts, + "ECMAScript line starts diverge on {label} {source:?}" + ); + assert_eq!( + lf_tracker.line_starts, + LocationTracker::new(source).line_starts, + "LF line starts diverge on {label} {source:?}" + ); + } + + #[test] + fn test_delta_map_matches_dense_reference_exhaustive() { + // An alphabet covering every arm of the builder: plain ASCII, both + // ASCII line terminators (so CR, LF, and CRLF pairs all occur), a + // 2-byte char, a 3-byte char, both multibyte line terminators, and an + // astral char (the 4-byte / surrogate-pair arm). Exhaustive over every + // string of length 0..=3 — enough to place any two arms adjacent, on + // either side of a run boundary, and at the start/end of the source. + const ALPHABET: [char; 8] = ['a', '\n', '\r', 'é', '中', '\u{2028}', '\u{2029}', '😀']; + + let mut source = String::new(); + assert_map_matches_reference(&source, "len 0"); + for &a in &ALPHABET { + for &b in &ALPHABET { + for &c in &ALPHABET { + for len in 1..=3 { + source.clear(); + source.push(a); + if len > 1 { + source.push(b); + } + if len > 2 { + source.push(c); + } + assert_map_matches_reference(&source, "exhaustive"); + } + } + } + } + } + + #[test] + fn test_delta_map_widens_at_the_u8_boundary() { + // Each 'é' grows the delta by exactly 1, so the source length in chars + // *is* the final delta — the narrow/wide boundary lands exactly at 255. + for (chars, expect_wide) in [(254, false), (255, false), (256, true), (700, true)] { + let source = "é".repeat(chars); + let map = ByteToCharMap::new(&source); + assert_eq!( + matches!(map.deltas, Deltas::Wide(_)), + expect_wide, + "wrong element width at {chars} chars" + ); + assert_map_matches_reference(&source, "u8 boundary"); + } + } + + #[test] + fn test_delta_map_wide_path_covers_every_arm() { + // The widening rebuild re-runs the whole scan, including the line + // rules — drive it with a source long enough to overflow `u8` that + // still contains every arm (a 3-byte char grows the delta by 2, an + // astral char by 2). + let source = "中a\r\n😀\u{2028}é\u{2029}x\r".repeat(60); + let map = ByteToCharMap::new(&source); + assert!(matches!(map.deltas, Deltas::Wide(_)), "expected wide table"); + assert_map_matches_reference(&source, "wide, all arms"); + } + + #[test] + fn test_delta_map_matches_dense_reference_on_long_mixed_sources() { + // The exhaustive test's strings are too short to reach the + // word-at-a-time run scan or a multi-run fill. These are long enough + // for both, at three multibyte densities: sparse (the real-corpus + // shape — long ASCII runs between characters, narrow table), mixed, + // and dense (wide table). Deterministic LCG, so a failure reproduces. + const ALPHABET: [char; 8] = ['a', ' ', '\n', '\r', 'é', '中', '\u{2028}', '😀']; + for (multibyte_in, expect_wide) in [(1000u32, false), (64, true), (8, true), (2, true)] { + let mut state = 0x2545_F491_4F6C_DD1Du64; + let mut source = String::new(); + while source.len() < 40_000 { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1); + let roll = (state >> 33) as u32; + if roll.is_multiple_of(multibyte_in) { + source.push(ALPHABET[4 + (roll % 4) as usize]); + } else { + source.push(ALPHABET[(roll % 4) as usize]); + } + } + assert_eq!( + matches!(ByteToCharMap::new(&source).deltas, Deltas::Wide(_)), + expect_wide, + "wrong element width at 1-in-{multibyte_in} density" + ); + assert_map_matches_reference(&source, "long mixed"); + } + } + + #[test] + fn test_next_non_ascii_finds_the_first_high_byte() { + // Every alignment within and past the word-at-a-time stride. + for prefix in 0..24usize { + let mut source = "a".repeat(prefix); + source.push('é'); + source.push_str("bcd"); + let bytes = source.as_bytes(); + assert_eq!(next_non_ascii(bytes, 0), prefix, "prefix {prefix}"); + // A scan starting past the character finds no further high byte. + assert_eq!( + next_non_ascii(bytes, prefix + 2), + bytes.len(), + "tail scan, prefix {prefix}" + ); + } + assert_eq!(next_non_ascii(b"", 0), 0); + assert_eq!(next_non_ascii(b"abc", 0), 3); + } + #[test] fn test_new_counts_lf_only() { // Svelte's locate-character convention: CR, U+2028, and U+2029 are not @@ -563,9 +941,12 @@ mod tests { } #[test] - fn test_new_ecmascript_with_map_matches_separate_builds() { - // Mixed content: CRLF, lone CR, U+2028, multibyte inside and at line - // boundaries, astral char — every branch of the fused scan. + fn test_fused_constructors_match_separate_builds() { + // Both fused constructors must stay byte-identical to the separate + // `new_ecmascript`/`new` + `ByteToCharMap::new` builds they replaced — + // which is exactly what `assert_map_matches_reference` asserts, on both + // halves. Hand-picked mixed sources: CRLF, lone CR, U+2028, multibyte + // inside and at line boundaries, astral, all-ASCII, and empty. for source in [ "abc", "a\r\nb\rc\nd", @@ -573,50 +954,7 @@ mod tests { "\u{2028}\r\n😀", "", ] { - let (tracker, map) = LocationTracker::new_ecmascript_with_map(source); - let expected_tracker = LocationTracker::new_ecmascript(source); - let expected_map = ByteToCharMap::new(source); - assert_eq!( - tracker.line_starts, expected_tracker.line_starts, - "line starts diverge on {source:?}" - ); - assert_eq!(map.has_multibyte(), expected_map.has_multibyte()); - for b in 0..=(source.len() as u32 + 2) { - assert_eq!( - map.byte_to_char(b), - expected_map.byte_to_char(b), - "map diverges at byte {b} on {source:?}" - ); - } - } - } - - #[test] - fn test_new_with_map_matches_separate_builds() { - // LF-only: CR / U+2028 / U+2029 are NOT line starts; multibyte inside and - // at line boundaries; astral char — the map half must match `new` + `new`. - for source in [ - "abc", - "a\r\nb\rc\nd", - "aé\r\né😀\u{2028}x\ry\n中", - "\u{2028}\r\n😀", - "", - ] { - let (tracker, map) = LocationTracker::new_with_map(source); - let expected_tracker = LocationTracker::new(source); - let expected_map = ByteToCharMap::new(source); - assert_eq!( - tracker.line_starts, expected_tracker.line_starts, - "LF-only line starts diverge on {source:?}" - ); - assert_eq!(map.has_multibyte(), expected_map.has_multibyte()); - for b in 0..=(source.len() as u32 + 2) { - assert_eq!( - map.byte_to_char(b), - expected_map.byte_to_char(b), - "map diverges at byte {b} on {source:?}" - ); - } + assert_map_matches_reference(source, "mixed"); } } diff --git a/docs/performance.md b/docs/performance.md index 86093064b..a6550528b 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -147,6 +147,15 @@ cargo run --release -p tsv_debug -- json_profile ~/dev/zzz/src/lib --json Output shows, per language: file/byte/wire-byte/multibyte counts and the `parse` and `write` medians (sums of per-file medians). +**When A/B-ing a write-path change, read `write` from here and `parse` from +`profile` (§1) — not from this command.** Both phases run in one process against +one allocator, so a change that alters what the writer allocates also changes the +state the *next* iteration's `parse` starts from; that alone can move this +command's `parse` median by a few percent in either direction while the parse +code is untouched. `profile` (§1) never calls `convert_ast_json_bytes` at all, +which makes it both the trustworthy parse surface and the **null control** for a +write-path change: a genuine write-path win leaves its totals flat. + ### 3. `[profile.profiling]` — cargo profile for perf The release profile strips debug symbols (`strip = true`), making `perf` useless. The `profiling` profile keeps symbols at release speed: