From 82112bda285e43ffe806b47cf1401955ad674d7f Mon Sep 17 00:00:00 2001 From: Andrei Demian Date: Fri, 4 Sep 2026 16:13:25 +0300 Subject: [PATCH 1/6] update: tests & performance --- .claude/workflows/library-review.js | 309 ++ .github/workflows/ci.yml | 135 +- .github/workflows/release.yml | 188 +- CHANGELOG.md | 338 +++ README.md | 130 +- SelectableMarkdown.podspec | 55 +- android/CMakeLists.txt | 26 +- android/build.gradle | 28 +- .../selectablemarkdown/RunAccessibility.kt | 572 ++++ .../selectablemarkdown/RunAttributedText.kt | 267 +- .../com/selectablemarkdown/RunDecorations.kt | 89 + .../java/com/selectablemarkdown/RunEmbeds.kt | 132 +- .../com/selectablemarkdown/RunLayoutCache.kt | 23 +- .../com/selectablemarkdown/RunTextMeasure.kt | 168 +- .../com/selectablemarkdown/RunTypefaces.kt | 149 + .../SelectableMarkdownModule.kt | 96 +- .../SelectableRunHostView.kt | 732 ++++- .../SelectableRunHostViewManager.kt | 97 +- .../SelectionChangeEvent.kt | 67 + android/src/main/jni/CMakeLists.txt | 34 +- android/src/main/jni/RNSMRunTextMeasurer.cpp | 5 +- android/src/main/res/values/strings.xml | 31 + bench/crossing.mjs | 11 +- bench/gates.test.ts | 336 +++ bench/head-to-head.mjs | 12 + bench/pathological.mjs | 350 ++- bench/projection.mjs | 302 ++ bench/streaming-replay.mjs | 477 +++- bench/support.mjs | 40 + bench/throughput.mjs | 20 +- .../fixtures/transcript-giant-list.json | 2490 +++++++++++++++++ .../selection/incremental-projection.test.ts | 293 ++ .../selection/projection-oracle.test.ts | 449 ++- conformance/streaming/prefix-oracle.test.ts | 487 +++- docs/ARCHITECTURE.md | 60 +- docs/BENCHMARKS.md | 237 +- docs/FABRIC-PLAN.md | 186 +- docs/NATIVE.md | 160 +- docs/PERFORMANCE.md | 242 +- docs/SELECTION.md | 881 +++++- docs/STREAMING.md | 510 +++- jest.config.js | 6 + native/node/README.md | 21 +- native/node/addon.cpp | 5 + native/node/index.mjs | 14 +- package-lock.json | 2 +- package.json | 64 +- platform/cpp/FlatBuffer.cpp | 39 +- platform/cpp/OffsetParser.cpp | 58 +- platform/cpp/OffsetParser.h | 12 +- platform/cpp/Protocol.h | 34 +- platform/cpp/SelectableMarkdownJsi.cpp | 15 +- platform/cpp/SelectableMarkdownJsi.h | 28 +- platform/cpp/vendor/md4c/UPSTREAM.md | 19 +- platform/fabric/RNSMRunHostShadowNode.h | 22 +- platform/fabric/RNSMRunTextMeasurer.h | 20 + platform/ios/RNSMAttributedText+Props.h | 26 +- platform/ios/RNSMAttributedText.h | 109 +- platform/ios/RNSMAttributedText.mm | 443 ++- platform/ios/RNSMTextSplice.swift | 312 +++ platform/ios/SelectableMarkdownModule.h | 37 +- platform/ios/SelectableMarkdownModule.mm | 104 +- platform/ios/SelectableRunHostView.swift | 1222 ++++++-- .../RCTSelectableRunHostComponentView.h | 78 +- .../RCTSelectableRunHostComponentView.mm | 108 +- platform/ios/fabric/RNSMRunTextMeasurer.mm | 82 +- scripts/changelog-section.mjs | 90 + scripts/check-codegen.mjs | 187 +- scripts/check-fabric-cpp.mjs | 146 +- scripts/check-lock-sync.mjs | 151 + scripts/check-swift.mjs | 33 +- scripts/check-unreleased-breaking.mjs | 188 ++ scripts/emit-dist-spec-shim.mjs | 184 +- scripts/finish-esm-build.mjs | 161 ++ scripts/packaging.test.ts | 574 ++++ scripts/release.mjs | 158 +- scripts/verify-pack.mjs | 324 ++- src/agui/agUiHooks.test.ts | 502 ++++ src/agui/bindRunTextEvents.test.ts | 529 +++- src/agui/bindRunTextEvents.ts | 441 ++- src/agui/useAgUiSession.test.ts | 317 +++ src/agui/useAgUiSession.ts | 501 +++- src/document/document.test.ts | 102 + src/document/nodes.ts | 6 +- src/document/visit.ts | 76 +- src/engine/Engine.ts | 9 + src/engine/entities.test.ts | 84 +- src/engine/entities.ts | 70 +- src/engine/extensions/spoilers.test.ts | 154 +- src/engine/extensions/spoilers.ts | 279 +- src/engine/native.ts | 36 +- src/engine/native/__tests__/documents.test.ts | 163 ++ .../native/__tests__/hostBinding.test.ts | 156 +- src/engine/native/__tests__/protocol.test.ts | 197 ++ src/engine/native/__tests__/spans.test.ts | 351 ++- src/engine/native/decode.ts | 295 +- src/engine/native/index.ts | 31 +- src/engine/native/install.ts | 112 +- src/engine/native/protocol.ts | 56 +- src/engine/native/widen.ts | 52 +- src/engine/options.test.ts | 110 + src/engine/options.ts | 136 +- src/engine/urlPolicy.test.ts | 190 +- src/engine/urlPolicy.ts | 137 +- src/index.test.ts | 263 ++ src/index.ts | 379 ++- src/selection/__tests__/copy.test.ts | 110 + src/selection/__tests__/mapSelection.test.ts | 858 +++++- src/selection/__tests__/runs.test.ts | 593 +++- src/selection/copy.ts | 93 +- src/selection/mapSelection.ts | 1152 +++++++- src/selection/runs.ts | 433 ++- src/stream/StreamSession.test.ts | 359 ++- src/stream/StreamSession.ts | 1344 +++++++-- src/stream/buffering.test.ts | 630 ++++- src/stream/clusters.test.ts | 251 ++ src/stream/clusters.ts | 297 ++ src/stream/incremental.test.ts | 375 ++- src/stream/placeholders.ts | 219 +- src/stream/repair.test.ts | 1283 ++++++++- src/stream/repair.ts | 1683 ++++++++++- src/stream/shiftSpans.test.ts | 79 + src/stream/shiftSpans.ts | 103 +- src/stream/smoothing.test.ts | 203 ++ src/stream/smoothing.ts | 216 +- src/view/RunHost.tsx | 491 +++- src/view/SelectableMarkdown.tsx | 1420 +++++++++- src/view/SelectableRunHostNativeComponent.ts | 279 +- src/view/imageEmbeds.test.ts | 232 ++ src/view/imageEmbeds.ts | 129 + src/view/processedColors.test.ts | 93 + src/view/processedColors.ts | 79 + src/view/projectionCache.test.ts | 333 +++ src/view/projectionCache.ts | 101 + src/view/propContracts.test.ts | 340 +++ src/view/renderers.test.ts | 428 +++ src/view/renderers.tsx | 467 +++- src/view/runAttributes.test.ts | 544 +++- src/view/runAttributes.ts | 727 ++++- src/view/runDecorations.test.ts | 50 + src/view/runEmbeds.test.ts | 53 +- src/view/runEmbeds.ts | 68 +- src/view/runIdentity.test.ts | 295 ++ src/view/runIdentity.ts | 185 ++ src/view/runPressables.test.ts | 38 + src/view/runPressables.ts | 32 +- src/view/selectionActionWire.test.ts | 285 ++ src/view/selectionActions.test.ts | 195 +- src/view/selectionActions.ts | 403 ++- src/view/selectionRange.test.ts | 359 +++ src/view/selectionRange.ts | 174 ++ src/view/theme.test.ts | 120 +- src/view/theme.ts | 179 +- tsconfig.build.json | 19 +- tsconfig.esm.json | 34 + tsconfig.json | 8 +- 156 files changed, 37647 insertions(+), 2818 deletions(-) create mode 100644 .claude/workflows/library-review.js create mode 100644 CHANGELOG.md create mode 100644 android/src/main/java/com/selectablemarkdown/RunAccessibility.kt create mode 100644 android/src/main/java/com/selectablemarkdown/RunTypefaces.kt create mode 100644 android/src/main/java/com/selectablemarkdown/SelectionChangeEvent.kt create mode 100644 android/src/main/res/values/strings.xml create mode 100644 bench/gates.test.ts create mode 100644 bench/projection.mjs create mode 100644 conformance/fixtures/transcript-giant-list.json create mode 100644 conformance/selection/incremental-projection.test.ts create mode 100644 platform/ios/RNSMTextSplice.swift create mode 100644 scripts/changelog-section.mjs create mode 100644 scripts/check-lock-sync.mjs create mode 100644 scripts/check-unreleased-breaking.mjs create mode 100644 scripts/finish-esm-build.mjs create mode 100644 scripts/packaging.test.ts create mode 100644 src/agui/agUiHooks.test.ts create mode 100644 src/index.test.ts create mode 100644 src/stream/clusters.test.ts create mode 100644 src/stream/clusters.ts create mode 100644 src/view/imageEmbeds.test.ts create mode 100644 src/view/imageEmbeds.ts create mode 100644 src/view/processedColors.test.ts create mode 100644 src/view/processedColors.ts create mode 100644 src/view/projectionCache.test.ts create mode 100644 src/view/projectionCache.ts create mode 100644 src/view/propContracts.test.ts create mode 100644 src/view/renderers.test.ts create mode 100644 src/view/runIdentity.test.ts create mode 100644 src/view/runIdentity.ts create mode 100644 src/view/selectionActionWire.test.ts create mode 100644 src/view/selectionRange.test.ts create mode 100644 src/view/selectionRange.ts create mode 100644 tsconfig.esm.json diff --git a/.claude/workflows/library-review.js b/.claude/workflows/library-review.js new file mode 100644 index 0000000..daf0783 --- /dev/null +++ b/.claude/workflows/library-review.js @@ -0,0 +1,309 @@ +export const meta = { + name: 'library-review', + description: 'Audit react-native-selectable-markdown: docs vs implementation, gaps, design improvements; adversarially verify every finding', + phases: [ + { title: 'Find', detail: '13 docs-vs-impl auditors + 17 subsystem gap finders' }, + { title: 'Merge', detail: 'dedupe findings per area' }, + { title: 'Verify', detail: 'adversarial fact-check, then impact judgment' }, + { title: 'Critic', detail: 'completeness critic, targeted follow-up finders' }, + { title: 'Synthesize', detail: 'prioritize and group' }, + ], +} + +const REPO = '/Users/demian/Work/superpower/react-native-selectable-markdown' +const SCRATCH = '/private/tmp/claude-501/-Users-demian-Work-superpower-react-native-selectable-markdown/74aacc70-da89-4d44-b104-cefc1375f952/scratchpad' +const AREAS = ['streaming','selection','view','native-ios','native-android','native-cpp-engine','engine-options','agui','packaging-ci-tests','docs-general','architecture-design'] + +const GOALS = 'a performant markdown renderer for streamed LLM output in React Native, with selection that spans paragraphs and other blocks ("document-grade" selection) and copies the exact markdown source; safe defaults for untrusted model output.' + +const PREAMBLE = `You are reviewing the git repository at ${REPO} (branch main, package version 0.11.0). It is \`react-native-selectable-markdown\`. Its stated goal: ${GOALS} +Layout: parsing is md4c compiled natively (platform/cpp, vendored md4c under platform/cpp/vendor) exposed over JSI (src/engine/native); streaming in src/stream; selection runs/projection/copy in src/selection; React view layer in src/view; native hosts in platform/ios (Swift + ObjC++), android/ (Kotlin + JNI C++), and a shared Fabric shadow node in platform/fabric. Docs: README.md and docs/*.md, plus native/node/README.md. Node test harness: native/node (an addon already built at build/selectable-markdown.*.node; all 32 Jest suites currently pass). Run tests with \`npx jest \` from the repo root; typecheck with \`npx tsc --noEmit\`. Do NOT run scripts/build-node-addon.mjs (the addon is built; concurrent builds would clash). +Rules: the repository is READ-ONLY for you. Never edit, create or delete files under ${REPO}. Put scratch scripts and repros under ${SCRATCH} (create a subdirectory named after your label). Read the actual code before claiming anything; every finding needs a file:line you verified by reading. No speculation presented as fact; if you could not verify something, say so in coverage_notes instead of reporting it. Do not restate the docs' own admitted gaps ("Known divergence", "known gap", the README Status table) unless you add something material, e.g. the gap is larger than described, or the description of it is wrong.` + +const FINDER_OUTPUT_RULES = `## Output rules +- Return findings only with verified file:line refs (code_ref) that you read in this session. For docs-vs-impl findings also fill doc_ref (doc file:line). +- One finding per distinct issue. Do not pad. An empty list with honest coverage_notes is a valid result. +- claim = what the doc, comment or API promises (or, for gaps, what the goals require); reality = what the code does; evidence = short verbatim quotes (one or two lines each) from both sides plus any repro output. +- severity: high = misleads a user into a wrong integration, or a real bug or performance cliff on the streaming or selection path; medium = wrong or missing information a maintainer should fix, or a real gap with a workaround; low = minor drift. +- category: docs-vs-impl (doc and code disagree), bug (code is wrong on its own terms), gap (something the goals need that is missing or partial), improvement (a design alternative that would be clearly better; state the trade-off). +- area: the subsystem the finding is about, from: ${AREAS.join(', ')}. +- coverage_notes: files read fully, files skimmed, checks you could not perform.` + +const FINDING_PROPS = { + title: { type: 'string', description: 'specific, under 100 chars' }, + category: { type: 'string', enum: ['docs-vs-impl', 'bug', 'gap', 'improvement'] }, + area: { type: 'string', enum: AREAS }, + severity: { type: 'string', enum: ['high', 'medium', 'low'] }, + doc_ref: { type: 'string', description: 'doc file:line, or empty string' }, + code_ref: { type: 'string', description: 'file:line(s) you read' }, + claim: { type: 'string' }, + reality: { type: 'string' }, + recommendation: { type: 'string' }, + evidence: { type: 'string' }, +} +const FINDING_REQUIRED = ['title', 'category', 'area', 'severity', 'doc_ref', 'code_ref', 'claim', 'reality', 'recommendation', 'evidence'] + +const FINDINGS_SCHEMA = { + type: 'object', + properties: { + findings: { type: 'array', items: { type: 'object', properties: FINDING_PROPS, required: FINDING_REQUIRED } }, + coverage_notes: { type: 'string' }, + }, + required: ['findings', 'coverage_notes'], +} + +const MERGED_SCHEMA = { + type: 'object', + properties: { + merged: { type: 'array', items: { type: 'object', properties: { ...FINDING_PROPS, source_ids: { type: 'array', items: { type: 'integer' } } }, required: [...FINDING_REQUIRED, 'source_ids'] } }, + }, + required: ['merged'], +} + +const FACT_SCHEMA = { + type: 'object', + properties: { + confirmed: { type: 'boolean' }, + confidence: { type: 'string', enum: ['high', 'medium', 'low'] }, + reasoning: { type: 'string', description: 'what you read or ran, and why the claim stands or falls' }, + corrected_title: { type: 'string', description: 'empty if the title is accurate' }, + corrected_code_ref: { type: 'string', description: 'empty if accurate' }, + corrected_doc_ref: { type: 'string', description: 'empty if accurate' }, + corrected_reality: { type: 'string', description: 'empty if accurate; otherwise the corrected statement of what the code does' }, + }, + required: ['confirmed', 'confidence', 'reasoning', 'corrected_title', 'corrected_code_ref', 'corrected_doc_ref', 'corrected_reality'], +} + +const MAT_SCHEMA = { + type: 'object', + properties: { + matters: { type: 'boolean' }, + already_documented: { type: 'boolean' }, + severity: { type: 'string', enum: ['high', 'medium', 'low'] }, + effort: { type: 'string', enum: ['small', 'medium', 'large'] }, + rationale: { type: 'string' }, + sharpened_recommendation: { type: 'string' }, + }, + required: ['matters', 'already_documented', 'severity', 'effort', 'rationale', 'sharpened_recommendation'], +} + +const CRITIC_SCHEMA = { + type: 'object', + properties: { + assessment: { type: 'string' }, + followups: { type: 'array', items: { type: 'object', properties: { key: { type: 'string' }, area: { type: 'string', enum: AREAS }, prompt: { type: 'string' } }, required: ['key', 'area', 'prompt'] } }, + }, + required: ['assessment', 'followups'], +} + +const SYNTH_SCHEMA = { + type: 'object', + properties: { + executive_summary: { type: 'string' }, + headline_ids: { type: 'array', items: { type: 'string' } }, + themes: { type: 'array', items: { type: 'object', properties: { title: { type: 'string' }, narrative: { type: 'string' }, finding_ids: { type: 'array', items: { type: 'string' } } }, required: ['title', 'narrative', 'finding_ids'] } }, + quick_wins: { type: 'array', items: { type: 'string' } }, + docs_fix_list: { type: 'array', items: { type: 'string' } }, + scope_notes: { type: 'string' }, + }, + required: ['executive_summary', 'headline_ids', 'themes', 'quick_wins', 'docs_fix_list', 'scope_notes'], +} + +const finders = [ + // ---------------- docs vs implementation ---------------- + { key: 'D01-readme-install', area: 'docs-general', prompt: `Audit README.md sections "Why", "Features", "Install" and "Native setup" (plus the two intro paragraphs) against the implementation. For each factual claim locate the code that backs it: installNativeEngine return semantics and the JavaScriptCore refusal (src/engine/native/install.ts, src/engine/native/index.ts), parseDocument throwing without the module (src/engine/native.ts, src/engine/Engine.ts), SelectableMarkdown throwing during render, RunHost throwing when the Fabric component is missing and the use_frameworks! claim (src/view/RunHost.tsx, src/view/SelectableRunHostNativeComponent.ts), autolinking (react-native.config.js, SelectableMarkdown.podspec, android/build.gradle, android/src/main/java/com/selectablemarkdown/SelectableMarkdownPackage.kt), the peer range in package.json, the CommonMark score (conformance/run-commonmark.mjs), "settled blocks keep referential identity" and "plain-prose deltas skip the parser" (src/stream/StreamSession.ts), "HTML is stripped, URL schemes are allowlisted" defaults (src/engine/options.ts, src/engine/urlPolicy.ts), "A run ends at an image, a spoiler, or a block you mark standalone" and "Code blocks, tables and rules flow through runs" (src/selection/runs.ts), "Copy gives you the markdown itself" (src/selection/copy.ts). Report anything wrong, stale, over-promised or under-described.` }, + { key: 'D02-readme-quickstart', area: 'docs-general', prompt: `Audit README.md "Quick start" (Static, Streaming, ag-ui, Headless) against src/view/SelectableMarkdown.tsx, src/stream/StreamSession.ts, src/stream/smoothing.ts, src/agui/useAgUiSession.ts, src/agui/bindRunTextEvents.ts, src/document/visit.ts and src/index.ts. Check every prop, option name, default value, function signature, return type and behavioral claim in the snippets and prose: selectionActions default and "system Copy always stays", onSelectionCopy payload fields (action, plain, markdown, span), "Setting one without the other yields an empty custom menu", holdBackChars and holdIdleMs defaults, appendBuffered/flushBuffered semantics, "any synchronous call (append, replace, finalize) drains first", the StreamSession option list (smoother, repair, bufferScheduler, idleScheduler, now), createSmoother and createAdaptiveSmoother signatures and described behaviour, drained(), rewrite(full), notifyRunFinalized, finalize reasons and idempotency, useAgUiSession signature and its finalize triggers, bindRunTextEvents/useAgUiRunSessions/holding, the Node deep-path import advice (does dist/engine/Engine exist after build? does the package root really re-export react-native?). Would the snippets typecheck and behave as described? Write a scratch .ts file under the scratchpad that imports from the package src and run \`npx tsc --noEmit\` on it if useful.` }, + { key: 'D03-readme-theming-options', area: 'docs-general', prompt: `Audit README.md "Theming", "Defaults for model output" (the option table), "Custom link schemes", the embed section and the classifyBlock section against src/view/theme.ts, src/engine/options.ts, src/engine/urlPolicy.ts, src/engine/extensions/spoilers.ts, src/view/renderers.tsx, src/view/runEmbeds.ts, src/view/SelectableMarkdown.tsx, src/selection/runs.ts, and the option handling on the native side (platform/cpp/OffsetParser.cpp, platform/cpp/SelectableMarkdownJsi.cpp, src/engine/native/index.ts). Verify: the theme group list, "merge one level deep", colorScheme default and behaviour, glyphs shifting offsets, attributeForMark taking part in the per-run memo, the "Known divergence" about listIndent, every option's default in commonmark/llmChat/everything presets, DEFAULT_LINK_PREFIXES, the image scheme default, blockedLinks 'text' vs 'node', onLinkPress payload and the Linking.openURL fallback, "the link renderer does not run inside a run", the embed contract (one placeholder character, declared size, topLevel, text for copy-text, onEmbedLayout, taps owned by the card), classifyBlock semantics and "Images and spoilers are standalone already", "the document is resegmented when it changes".` }, + { key: 'D04-readme-status-bench', area: 'docs-general', prompt: `Audit README.md "Architecture" diagram, "Status (0.11.x)" table, "Benchmarks" paragraph, "Contributing" and "Prior art" against the repository: package.json version and scripts, bench/*.mjs (do the harnesses exist and produce the quoted metrics: "at most 277 characters", "2.7 µs", "43 to 53%", "15.8 MB/s"), docs/BENCHMARKS.md numbers vs README numbers, conformance/run-commonmark.mjs and conformance/vendor (651/652, example 174), the copy-menu iOS version claims vs platform/ios/SelectableRunHostView.swift, "Fabric only" vs platform/ and android/, "no example app", the "Removed in 0.10.0, restored in 0.11.0" history (git log has two commits; is there a CHANGELOG?), "CI compiles the C++ against real renderer headers and the Swift against the iOS SDK" vs .github/workflows/ci.yml and scripts/check-fabric-cpp.mjs, scripts/check-swift.mjs, "iOS preserves it" (selection), and the Contributing commands. You may run \`node bench/streaming-replay.mjs\` to spot-check the shape of its output. Report mismatches and unverifiable claims.` }, + { key: 'D05-architecture-doc', area: 'docs-general', prompt: `Audit docs/ARCHITECTURE.md against the code: the pipeline description, the "three rules", the module map (every file or directory it names must exist and do what it says; list significant files that exist but are missing from the map), the Engine interface as documented vs src/engine/Engine.ts (exact type), "Writing an engine" (is SourceSpan really the only hard requirement? find places in src/stream, src/selection and src/view that assume md4c-specific behaviour: node kinds, incomplete/synthetic flags, entity handling, smart punctuation, span widening), "The engine that ships", "Headless use from Node".` }, + { key: 'D06-streaming-doc', area: 'streaming', prompt: `Audit docs/STREAMING.md against src/stream/StreamSession.ts, repair.ts, smoothing.ts, placeholders.ts, shiftSpans.ts and the tests (incremental.test.ts, buffering.test.ts, repair.test.ts, StreamSession.test.ts, smoothing.test.ts, conformance/streaming/prefix-oracle.test.ts). Verify every claim: session lifecycle and states, pacing (holdback, idle flush, buffering), the Smoother contract, the safe anchor definition and the tail-only reparse rules, the construct-free fast path conditions, the tail repair handler table (does each documented handler exist with the documented behaviour; count the corpus, the README says 147 cases), the incomplete and synthetic flags, finalize semantics for 'end' | 'aborted' | 'failed', and the protocol edge cases. Report every mismatch, plus behaviour in the code that the doc omits and a user would need to know.` }, + { key: 'D07-selection-doc-1', area: 'selection', prompt: `Audit docs/SELECTION.md sections "Pipeline", "Runs and standalone blocks", "Projection", "Native component" (including "How RunHost resolves the component" and "Status"), "Props", "Event: onSelectionAction", "Event: onInlinePress", "Event: onEmbedLayout" and "handleSelectionAction" against src/selection/runs.ts, src/selection/mapSelection.ts, src/selection/copy.ts, src/view/RunHost.tsx, src/view/SelectableRunHostNativeComponent.ts, src/view/selectionActions.ts, src/view/runAttributes.ts, runDecorations.ts, runPressables.ts, runEmbeds.ts. Check each documented prop and event name, type, default, and the projection rules (which blocks flow, which are standalone, glyphs, code fences, tables, rules, line breaks) against the code. Report mismatches and undocumented props or events.` }, + { key: 'D08-selection-doc-2', area: 'selection', prompt: `Audit docs/SELECTION.md sections "Selection preservation across text swaps" (iOS and Android), "View recycling (Fabric)", "Tail policy (streaming)", "What a standalone block does and does not get", "Sizing", "Android layout cache and spannable handoff", "Platform hardening" and "Offsets end to end" against platform/ios/SelectableRunHostView.swift, platform/ios/RNSMAttributedText.mm, platform/ios/RNSMTextKitStack.mm, platform/ios/fabric/RCTSelectableRunHostComponentView.mm, platform/ios/fabric/RNSMRunTextMeasurer.mm, platform/fabric/*.h and *.cpp, android/src/main/java/com/selectablemarkdown/*.kt, android/src/main/jni/RNSMRunTextMeasurer.cpp, and src/view/SelectableMarkdown.tsx (tail policy). Verify every described mechanism exists and behaves as written: clamping rules, recycling resets, cache keys and invalidation, the offsets pipeline (UTF-16 vs UTF-8 vs Java char vs NSString), sizing. Report mismatches.` }, + { key: 'D09-native-doc', area: 'native-cpp-engine', prompt: `Audit docs/NATIVE.md and native/node/README.md against platform/cpp/OffsetParser.cpp/.h, FlatBuffer.cpp, SelectableMarkdownJsi.cpp/.h, Protocol.h, platform/cpp/vendor/md4c/UPSTREAM.md and patches/README.md (are the described patches actually present in md4c.c? diff against the claimed upstream version if you can reason about it), src/engine/native/install.ts, decode.ts, widen.ts, protocol.ts, index.ts, native/node/addon.cpp, native/node/index.mjs, scripts/build-node-addon.mjs, android/src/main/cpp/OnLoad.cpp, platform/ios/SelectableMarkdownModule.mm. Verify: the wire format field by field (doc vs Protocol.h vs decode.ts), ownership claims, span widening rules, decoder hot paths, the "Behaviours that look like bugs and are not" list, build instructions, "Using it in an app", "When nothing renders" troubleshooting. Report mismatches.` }, + { key: 'D10-perf-bench-docs', area: 'architecture-design', prompt: `Audit docs/PERFORMANCE.md and docs/BENCHMARKS.md against the code and bench harnesses. For every optimization claimed in PERFORMANCE.md section 2 (delta coalescing, lookahead-by-lag holdback, tail repair hardening, ASCII fast path in the offset map, decoder hot paths, Android spannable handoff and measure cache, iOS append-only attributed text) find the implementing code and confirm it exists and does what is described. Check section 1 "Where the time goes" numbers against bench/crossing.mjs output shape, section 3 "Attempted, and not done" and section 4 roadmap against what is in the tree (anything on the roadmap already done? anything described as done that is not?). For BENCHMARKS.md: do bench/*.mjs implement the described methodology, do the metrics and gates named exist, are the numbers internally consistent with README.md, is the "Running" section accurate (flags, --libs). You may run \`node bench/streaming-replay.mjs\`, \`node bench/throughput.mjs\` and \`node bench/crossing.mjs\` (not head-to-head) to spot-check output shape, not exact numbers.` }, + { key: 'D11-fabric-plan-doc', area: 'architecture-design', prompt: `Audit docs/FABRIC-PLAN.md against the shipped code; it is described as a design retrospective. Check: section 0 and 0.1 (verification status, "what the plan got wrong") vs the code; section 2 (dual-architecture strategy, 2.1 spec resolution order, 2.2 podspec, 2.3 Gradle, 2.4 layout, 2.5 supported RN range) vs package.json peer (>=0.82), react-native.config.js, SelectableMarkdown.podspec, android/build.gradle, android/CMakeLists.txt, android/src/main/jni/CMakeLists.txt (does any old-architecture path still exist? is dual-arch text stale?); section 3 (sparse attributes, colours in JS, no flattening) vs src/view/SelectableRunHostNativeComponent.ts and platform/fabric prop decoding; section 4 (measurement agreement, 4.4 "the one line that makes streaming viable") vs RNSMRunTextMeasurer on both platforms and RNSMRunHostShadowNode; section 5 selection; section 6 residual styling fixed/cut items vs code; section 7 file-by-file plan vs actual files; section 8 verification vs scripts/check-*.mjs and CI; section 9 risks. Report where the document and the code disagree and where it is stale enough to mislead a maintainer.` }, + { key: 'D12-jsdoc-api-comments', area: 'docs-general', prompt: `Audit the inline documentation (JSDoc and comments on exported types, props and functions) of the public API against behaviour. Enumerate exports from src/index.ts (it uses export * so list what each module exports), then read src/engine/options.ts, src/engine/Engine.ts, src/stream/StreamSession.ts (options and methods), src/stream/smoothing.ts, src/stream/repair.ts (RepairOptions), src/stream/placeholders.ts, src/view/SelectableMarkdown.tsx (props), src/view/RunHost.tsx (props), src/view/theme.ts, src/view/renderers.tsx, src/agui/bindRunTextEvents.ts (RunBindingPolicy), src/agui/useAgUiSession.ts, src/selection/*.ts, src/document/*.ts. Report doc comments that describe behaviour the code does not have, defaults that differ from the comment, comments referencing removed things, and exported public API that has no documentation anywhere (neither JSDoc nor README nor docs/) but a consumer needs.` }, + { key: 'D13-ops-docs-scripts', area: 'packaging-ci-tests', prompt: `Audit the operational docs and scripts: conformance/vendor/README.md, platform/cpp/vendor/md4c/UPSTREAM.md and patches/README.md, .github/workflows/ci.yml and release.yml (the long explanatory comments vs the actual steps), scripts/release.mjs, scripts/verify-pack.mjs, scripts/check-codegen.mjs, scripts/check-fabric-cpp.mjs, scripts/check-swift.mjs, scripts/emit-dist-spec-shim.mjs, jest.config.js, tsconfig.json, tsconfig.build.json, package.json (files, main/types/react-native fields, scripts, devDependencies vs peerDependencies: react-native 0.75.4 in devDependencies vs peer >=0.82; @types/react 18 vs react >=18). Do the described procedures match what the scripts do? Do CI comments match CI steps (e.g. comments citing suite counts like "19 of the 31" or "ten of twenty-six" vs the actual 32 suites)? Does the release script do what release.yml expects? Would \`npm pack\` include everything the podspec and gradle need (compare podspec source_files and android sourceSets against package.json files; run \`npm pack --dry-run\` from the repo root, it does not modify the tree, and read the file list)?` }, + + // ---------------- gaps, bugs, improvements ---------------- + { key: 'G01-stream-session', area: 'streaming', prompt: `Deep-review src/stream/StreamSession.ts (and StreamSession.test.ts, buffering.test.ts) for correctness gaps, edge cases and performance problems on the streaming path. Cover: append/appendBuffered/flushBuffered/replace/rewrite/finalize/drained/notifyRunFinalized ordering and re-entrancy (a listener calling append during a notification); holdback and idle timers (leaks, timers firing after finalize or after the consumer drops the session, injectable schedulers); UTF-16 surrogate pairs and grapheme clusters split across deltas or across the holdback boundary; behaviour under React StrictMode double mount and unmount; the subscription API and memory; per-append cost (what work is proportional to the whole document rather than the delta: string concatenation, span shifting, node cloning, snapshot creation); error handling when the engine throws mid-stream; finalize reasons and what 'aborted' and 'failed' change. Prefer findings backed by a small repro you ran via \`npx jest\` (a scratch test file under the scratchpad with rootDir pointed at the repo, or a node script importing dist is NOT available; simplest is \`npx jest --rootDir ${REPO} \` with the repo's jest config) over speculation.` }, + { key: 'G02-incremental-anchor', area: 'streaming', prompt: `Deep-review the incremental parsing design: the safe anchor and tail-only reparse in src/stream/StreamSession.ts and helpers, src/stream/shiftSpans.ts, src/stream/placeholders.ts, plus conformance/streaming/prefix-oracle.test.ts and src/stream/incremental.test.ts. Hunt for inputs where a later delta can legally change the parse of an already settled block: link reference definitions that arrive later, setext heading underlines, lazy continuation lines, list tightness changing when a later item gains a blank line, list item numbering or start, tables whose delimiter row arrives later, fenced code that never closes or closes with a longer fence, HTML blocks, blockquote laziness, indented code after a list, spoilers and underline extensions, math when enabled, a trailing backslash or two-space hard break, CRLF. Determine whether the anchor logic handles each and whether the prefix oracle's fixtures (conformance/fixtures) would catch a miss. Write scratch repros that feed prefixes through StreamSession and compare with a fresh parse (run with \`npx jest --rootDir ${REPO} \`). Also assess the "construct-free fast path" for false positives: a delta the fast path accepts that actually changes the parse.` }, + { key: 'G03-tail-repair', area: 'streaming', prompt: `Deep-review src/stream/repair.ts (tail repair) and repair.test.ts. Enumerate the handlers. For each, look for false repairs (complete, legitimate text altered), missed cases that still flash (emphasis with underscores, nested emphasis, inline code with multiple backticks, links with titles, images, autolinks, strikethrough, spoilers, math when enabled, tables mid-row, ATX headings, setext underlines, thematic breaks vs list markers, fenced code with tildes, HTML), interaction with holdback, whether repair knows which extensions are enabled (an option-blind repair of \`~~\` or \`||\` when those extensions are off), the hideUriLikeLabels and hideBareUriSchemes behaviour, and the cost per delta (is repair O(tail) or O(document)? how is the tail located?). Back claims with repros: a scratch jest test run with \`npx jest --rootDir ${REPO} \`.` }, + { key: 'G04-smoothing-agui', area: 'agui', prompt: `Deep-review src/stream/smoothing.ts, src/agui/useAgUiSession.ts, src/agui/bindRunTextEvents.ts and their tests. Smoothers: createSmoother and createAdaptiveSmoother contracts, boundary handling ('word' boundaries with CJK or Thai text that has no spaces, surrogate pairs, markdown syntax boundaries such as releasing half of \`**\`), lag targets, the run-end drain deadline, timer leaks, behaviour when the app is backgrounded (timers paused, then a burst), whether rewrite() during smoothing is sound, and whether a smoother can starve (never drain) or overshoot. ag-ui: the structural event types accepted vs the ag-ui protocol (TEXT_MESSAGE_START/CONTENT/END, RUN_STARTED/FINISHED/ERROR, and also TEXT_MESSAGE_CHUNK, MESSAGES_SNAPSHOT, STATE_SNAPSHOT/DELTA, RAW, CUSTOM, STEP_*), hook lifecycle (session creation per messageId, cleanup on unmount, StrictMode double invoke, messageId change mid-stream, events object identity changing), RunBindingPolicy semantics, and the correctness of 'holding'. Report gaps and bugs with evidence.` }, + { key: 'G05-selection-mapping', area: 'selection', prompt: `Deep-review cross-block selection correctness in src/selection/runs.ts, src/selection/mapSelection.ts, src/selection/copy.ts, their tests under src/selection/__tests__, and conformance/selection/projection-oracle.test.ts. Focus: mapping display offsets back to source across block boundaries (paragraph to list to code block to table to rule and back); glyphs (bullets, task markers, ordered numbering, nested indentation); entity and escape decoding (\`&\`, \`\\\\*\`, numeric entities) where one source char count differs from display; smart punctuation; soft and hard line breaks; code fences (fence lines and info strings excluded from display but included in copy-markdown?); tables (cell separators, delimiter row, alignment, escaped pipes); nested lists and blockquotes; embeds (the placeholder char); surrogate pairs and combining marks at selection edges; selections that start or end inside syntax (inside \`**\` or a link's URL); selection spanning a standalone block boundary; whether copy-markdown yields sensible markdown when the selection starts mid-list-item or mid-table-row. Also performance: is projection recomputed for every run on every stream frame, and is any of it O(document) per token? Run the existing property tests and write scratch repros (\`npx jest --rootDir ${REPO} \`).` }, + { key: 'G06-view-layer', area: 'view', prompt: `Deep-review src/view/SelectableMarkdown.tsx, src/view/RunHost.tsx and src/view/renderers.tsx for gaps and React-level performance under streaming. Trace what re-renders per streamed frame: are settled runs memoized and how (React.memo? useMemo keyed on what?), are keys stable across resegmentation, do attribute/decoration/pressable/embed arrays get rebuilt and re-sent to native every frame for the whole document or only the changing run, does the tail get its own host and what happens at run boundaries (a new paragraph appended: is the previous run's native view retained or remounted?), behaviour inside FlatList/virtualized lists and with very long documents, error boundaries and missing-engine behaviour, prop identity footguns (theme/options/renderers/embed/classifyBlock objects created inline), accessibility (accessibilityRole, screen readers on the native host, dynamic type), RTL, the tail policy (what does an unsettled tail render as while a code block or table is open?), image handling, link press, standalone renderer coverage (every node kind rendered? what does an unknown kind render?), and the session prop (switching sessions, subscribing/unsubscribing, setState per flush vs useSyncExternalStore). Report concrete gaps with file:line.` }, + { key: 'G07-attributes-theme', area: 'view', prompt: `Deep-review src/view/runAttributes.ts, runDecorations.ts, runPressables.ts, runEmbeds.ts, theme.ts, selectionActions.ts and their tests. Questions: are attributes emitted as sparse ranges or per character; are overlapping marks (bold inside link inside heading, code inside a link) merged correctly and deterministically; decoration geometry for code blocks, tables and rules across line wraps and at run ends; theme merge depth and whether nested groups (headings per level, table borders, code font) can be partially overridden without clobbering; colorScheme 'auto' reactivity (Appearance listener or useColorScheme?); font scaling (allowFontScaling, maxFontSizeMultiplier); lineHeight semantics (multiplier vs px) consistency between JS, iOS and Android; attributeForMark memo correctness and cache growth; per-frame cost of rebuilding attributes for long runs; selection action ids and localisation of 'Copy Text' / 'Copy Markdown' labels; the color format sent to native (processColor?). Report gaps with evidence.` }, + { key: 'G08-ios-host', area: 'native-ios', prompt: `Deep-review the iOS host: platform/ios/SelectableRunHostView.swift, RNSMAttributedText.mm, RNSMAttributedText+Props.h, RNSMTextKitStack.mm, SelectableMarkdownModule.mm, platform/ios/fabric/RCTSelectableRunHostComponentView.mm, platform/ios/fabric/RNSMRunTextMeasurer.mm and the shared shadow node in platform/fabric. Focus: selection preservation across text swaps (does it survive attribute-only changes; does it clamp correctly when text shrinks on rewrite; does it survive view recycling; what about the selection handles and the edit menu during a swap); the append-only attributed text path (when taken; can it leave stale attributes when a mark spanning the boundary changes, e.g. \`**bold\` completing, or a heading's setext underline arriving); measurement agreement between the measurer used by Yoga and the UITextView layout (font fallback, emoji, line spacing, paragraph spacing, exclusion paths for embeds, width rounding, textContainerInset); main-thread cost per frame (relayout of the whole text storage per delta? any layout caching?); TextKit 1 vs 2 selection and iOS version gating; UIEditMenuInteraction vs UIMenuController paths and the iOS 13.4 to 15 behaviour; link taps vs selection gesture conflict; embed overlay hit testing and layout reporting timing; retain cycles; thread safety of props into the text storage; accessibility; dynamic type. Report concrete gaps with file:line.` }, + { key: 'G09-android-host', area: 'native-android', prompt: `Deep-review the Android host: android/src/main/java/com/selectablemarkdown/*.kt (SelectableRunHostView, SelectableRunHostViewManager, RunAttributedText, RunDecorations, RunEmbeds, RunLayoutCache, RunTextMeasure, the event classes, SelectableMarkdownModule, SelectableMarkdownPackage), android/src/main/jni/RNSMRunTextMeasurer.cpp, android/src/main/cpp/OnLoad.cpp, both CMakeLists.txt, android/build.gradle, and the Fabric state/measure path in platform/fabric. Focus: the selection loss on each text swap (what exactly drops it; what a fix would require: setText vs in-place Editable edits, Selection.setSelection restore, ActionMode survival); layout cache keys and invalidation correctness (width, density, font scale, theme change, text change); measurement agreement between the JNI measurer and TextView layout (StaticLayout params, hyphenation, break strategy, includeFontPadding, fallback fonts, emoji); the spannable handoff (built on which thread, cost per frame for long runs); ActionMode / copy menu customisation parity with iOS; link and embed touch handling; view recycling resets (prepareForRecycle equivalent); threading (state updates from the shadow thread vs UI thread); leaks; accessibility; minSdk and API level gating. Report concrete gaps with file:line.` }, + { key: 'G10-cpp-jsi', area: 'native-cpp-engine', prompt: `Deep-review platform/cpp/OffsetParser.cpp/.h, FlatBuffer.cpp, SelectableMarkdownJsi.cpp/.h, Protocol.h, native/node/addon.cpp, src/engine/native/decode.ts, widen.ts, install.ts, protocol.ts, index.ts and the tests under src/engine/native/__tests__. Focus: UTF-8 byte to UTF-16 offset mapping correctness (astral characters, invalid UTF-8, BOM, CRLF, lone surrogates in the JS source: how is the JS string converted to UTF-8 and back, is the round trip lossless?); md4c callback coverage (every MD_BLOCKTYPE and MD_SPANTYPE handled? every MD_TEXTTYPE: entities, nullchar, softbr, br, html, latexmath); buffer growth and integer overflow on large input; error paths (md_parse returning non-zero, allocation failure); ownership and lifetime of the ArrayBuffer returned to JS; thread affinity of the JSI host function (can it be called from a worklet or a background thread?); the JavaScriptCore refusal rationale; config and option encoding (does every EngineOptions field reach md4c? what md4c flags are not exposed: MD_FLAG_PERMISSIVEURLAUTOLINKS, PERMISSIVEWWWAUTOLINKS, LATEXMATHSPANS, WIKILINKS, HARD_SOFT_BREAKS, NOINDENTEDCODEBLOCKS, NOHTMLBLOCKS vs NOHTMLSPANS); the vendored md4c patches under platform/cpp/vendor/md4c/patches (applied? documented?); decode.ts hot path allocations. Write scratch repros against the addon (native/node/index.mjs) where possible. Report concrete gaps with file:line.` }, + { key: 'G11-engine-options-policy', area: 'engine-options', prompt: `Deep-review src/engine/options.ts, urlPolicy.ts, entities.ts, extensions/spoilers.ts, Engine.ts, native.ts, document/nodes.ts, document/visit.ts, document/span.ts and their tests. Focus: URL policy holes (scheme case-insensitivity, leading whitespace or control characters, \`javascript:\` and other entity-encoded forms as md4c delivers them, protocol-relative //, data: images, IDN and unicode, percent-encoded schemes, whether the policy is applied equally to autolinks, reference links, images and to mailto: with header parameters); html:'strip' semantics (is inline HTML removed or shown as text? does stripping preserve spans?); entity decoding vs the CommonMark entity table; the spoilers extension (nested, across emphasis, streaming with unbalanced ||, interaction with tables where | is a separator); node model completeness (node kinds without a renderer or a projection rule; footnotes, wikilinks, math node shapes); visit() API ergonomics; preset composition. Write scratch repros with the addon where helpful. Report concrete gaps.` }, + { key: 'G12-tests-ci', area: 'packaging-ci-tests', prompt: `Assess test coverage and CI for gaps relative to the goals (streaming performance, cross-block selection). Read jest.config.js, the test file list under src/ and conformance/, .github/workflows/ci.yml, scripts/check-codegen.mjs, check-fabric-cpp.mjs, check-swift.mjs. Identify: subsystems with no automated tests (React components, native iOS/Android behaviour beyond compile checks, JSI install path, ag-ui hooks under React), property tests present vs missing, whether the prefix oracle covers the extension options and the repair options and holdback combinations, whether any performance regression gate exists, whether the check-* scripts exercise the real code or only compile stubs, the devDependency react-native 0.75.4 vs peer >=0.82 (what does jest test against, what codegen version does check-codegen run, does the shipped spec depend on 0.82 behaviour?), and flaky or skipped tests. Run \`npx jest --listTests\` and \`npx jest\` and report the actual pass/skip state (note the stack trace printed from src/engine/native/__tests__/hostBinding.test.ts:152 during a passing run: what is it?). Report concrete gaps.` }, + { key: 'G13-packaging-build', area: 'packaging-ci-tests', prompt: `Assess packaging, build and consumer-integration gaps: package.json (files, main/types/react-native fields, absence of an exports map, sideEffects, prepare/prepack running tsc during a consumer's git install, engines), SelectableMarkdown.podspec (source_files, dependencies, install_modules_dependencies, new-arch flags, Swift and ObjC++ mixing, module map, header search paths, the platform/fabric/android-include exclusion), android/build.gradle and both CMakeLists.txt (RN version detection, prefab targets, namespace, minSdk, Kotlin version, codegen output dir, ABI filters), react-native.config.js, the codegenConfig (jsSrcsDir src/view: does codegen also try to parse non-spec files there?), scripts/emit-dist-spec-shim.mjs (what the dist spec shim is for), scripts/verify-pack.mjs, scripts/release.mjs and .github/workflows/release.yml. Check whether shipping TS via the react-native field while main points at dist is consistent for Metro, Expo and type resolution, and whether Expo config plugin needs exist. Run \`npm pack --dry-run\` (read-only) to inspect the file list. Report concrete gaps with file:line.` }, + { key: 'G14-perf-architecture', area: 'architecture-design', prompt: `Cross-cutting performance review against the goal "performant markdown renderer from streaming". Trace one streamed delta end to end: StreamSession.append (src/stream/StreamSession.ts) to engine parse (JSI crossing in src/engine/native/index.ts and platform/cpp/SelectableMarkdownJsi.cpp, FlatBuffer decode in src/engine/native/decode.ts) to document settle/diff to selection runs and projection (src/selection/runs.ts) to attributes/decorations/pressables/embeds (src/view/run*.ts) to React render (src/view/SelectableMarkdown.tsx, RunHost.tsx) to native props (src/view/SelectableRunHostNativeComponent.ts, platform/fabric props parsing and RNSMRunHostShadowNode) to shadow-node measure (RNSMRunTextMeasurer on both platforms) to the native view update (platform/ios/RNSMAttributedText.mm, SelectableRunHostView.swift; android RunAttributedText.kt, RunLayoutCache.kt). For each stage state what is proportional to the delta and what is proportional to the whole document or the whole tail run, with file:line evidence. Identify the largest wins not taken and design alternatives (per-run hosts so settled runs are never re-sent; sending deltas instead of full attribute arrays; a native-side projection; measure caching keyed on the settled prefix; avoiding double layout for measure and display; batching to frames; state-based commits vs props). Compare with how software-mansion/enriched-markdown, Expensify/react-native-live-markdown and vercel/streamdown approach the same problem where relevant (from your knowledge; do not fetch). Evaluate docs/PERFORMANCE.md's roadmap order against your analysis. Report as findings (category gap or improvement) with concrete evidence.` }, + { key: 'G15-selection-architecture', area: 'architecture-design', prompt: `Cross-cutting design review against the goal "cross paragraph selection". Evaluate the chosen architecture (one native text view per run; adjacent flowing blocks merged into a run; standalone blocks break runs) using src/selection/runs.ts, src/view/SelectableMarkdown.tsx, RunHost.tsx and the iOS and Android hosts. Assess: what still breaks selection continuity (images, spoilers, standalone-classified blocks, embeds, the streaming tail) and whether that is acceptable for a chat UI; behaviour with very long messages (one huge native text view: layout cost, memory, no virtualization, scroll performance); selection across multiple runs or across multiple messages in a chat list (is there any story for selecting across two SelectableMarkdown instances, as a document would allow?); selection during streaming (is the tail a separate run/host from the settled runs, and does that break selecting across the boundary? does selection survive the tail settling into a run?); copy-menu customisation limits (iOS 16+, Android ActionMode); hardware keyboard selection; accessibility. Propose concrete alternative designs with trade-offs (a single host for the whole document with block chrome as decorations; images and embeds via exclusion paths or attachments so they no longer split runs; a native-driven selection overlay spanning multiple views; Android in-place Editable updates). Report as findings (category gap or improvement) with evidence.` }, + { key: 'G16-api-dx', area: 'architecture-design', prompt: `Review the public API surface and developer experience. Read src/index.ts (it re-exports whole modules with export *: enumerate what leaks that looks internal), the props of SelectableMarkdown (src/view/SelectableMarkdown.tsx) and RunHost, StreamSession's API, the option and preset shapes (src/engine/options.ts), theme types, renderer override types (src/view/renderers.tsx), embed and classifyBlock contracts, event payloads. Look for: footguns (paired props; identity requirements for memoization; options that must be annotated to typecheck, as the README admits for blockedLinks: 'node'; a session that must be finalized to free timers), inconsistent naming, unclear defaults, missing escape hatches (custom selection actions beyond copy; controlled or programmatic selection; scroll-to-source-span; getting the display text; an imperative ref API; per-block renderers for flowing blocks), error messages when misused, TypeScript strictness (any, unsound casts, non-exhaustive switches), and API stability concerns for 0.11. Write a scratch consumer .tsx under the scratchpad and typecheck it against the package src to confirm any typing claim. Report concrete findings with file:line.` }, + { key: 'G17-security-robustness', area: 'engine-options', prompt: `Robustness and security review for untrusted model output. Read bench/pathological.mjs, src/engine/urlPolicy.ts, options.ts, src/stream/repair.ts, src/selection/runs.ts, platform/cpp/*.cpp and the native hosts. Look for: parser CPU blowups (md4c pathological cases: deeply nested brackets or emphasis, long runs of [, huge tables, many link reference definitions, pathological code spans) and whether anything bounds input size or nesting; JS-side quadratic paths in repair, runs, projection or attribute building on adversarial input (thousands of tiny blocks, a 1 MB single paragraph, a 100k-row table, a 10k-deep list); native crash vectors (out-of-range NSRange or Java index with emoji, flags, ZWJ sequences; negative lengths; attribute ranges beyond text length; decoration rects with NaN; embed sizes of 0 or Infinity; invalid UTF-8 from a JS string containing lone surrogates); URL policy bypasses; html:'raw' implications; memory growth in long sessions (caches without bounds: RunLayoutCache, iOS caches, JS memo maps, the repair corpus?). Write scratch repros against the addon (native/node/index.mjs) or via \`npx jest --rootDir ${REPO} \` where feasible, with timing. Report concrete findings.` }, +] + +function finderPrompt(f, extra) { + return `${PREAMBLE}\n\n## Your assignment (${f.key}; default area: ${f.area})\n${f.prompt}\n${extra || ''}\n${FINDER_OUTPUT_RULES}` +} + +function mergePrompt(area, items) { + return `${PREAMBLE}\n\n## Task\nYou are deduplicating review findings for the "${area}" area. Below are ${items.length} findings from several independent finders (JSON, each with an integer id). Merge findings that describe the same underlying issue (the same doc claim against the same code behaviour, or the same defect) into one, keeping the most precise refs, the strongest evidence from any duplicate, and the union of source_ids. Do NOT merge findings that share a file but describe different issues. Do not drop non-duplicates. Do not add new findings. Do not re-verify (that happens next). Keep severity as the maximum among merged duplicates. Output every surviving finding with all fields filled.\n\n\`\`\`json\n${JSON.stringify(items, null, 1)}\n\`\`\`` +} + +function factPrompt(f) { + return `${PREAMBLE}\n\n## Task: adversarial fact-check of one finding\nYour default is to refute. Open every referenced file and line (and the doc section, if any). Confirm only if you can quote the code (and the doc) that establishes the claim as stated. Refute if: the code actually behaves as the doc says; the doc actually says something else, or already documents the gap the finding presents as undocumented; the refs are wrong and you cannot locate the behaviour elsewhere; the finding rests on an assumption you could not verify; or the finding misattributes behaviour outside this repo's control. For docs-vs-impl findings, quote both sides. If it is partially right, set confirmed=true only if the core claim stands, and fill the corrected_* fields for the parts that were wrong (leave them empty strings when accurate). For behaviour claims that a test can settle, run \`npx jest \` or a scratch test (\`npx jest --rootDir ${REPO} \`) or a node script against the addon (native/node/index.mjs); a repro beats reading. Category "improvement" findings: confirm only if the description of the current design is accurate and the proposed alternative is technically coherent for React Native Fabric.\n\n\`\`\`json\n${JSON.stringify(f, null, 1)}\n\`\`\`` +} + +function matPrompt(f) { + return `${PREAMBLE}\n\n## Task: judge whether one (already fact-checked) finding matters\nRead the referenced files enough to judge impact; do not re-verify facts. Decide:\n- matters=false if it is a style nitpick, a hypothetical with no realistic trigger, advice that contradicts a deliberate documented decision without a stronger argument, or a restatement of something the docs already admit with nothing added (then also set already_documented=true).\n- severity: high = a user following the docs gets a wrong integration, a real bug, or a performance cliff on the streaming or selection path; medium = wrong or missing information a maintainer should fix, or a real gap with a workaround; low = minor drift.\n- effort: small (under an hour), medium (about a day), large (multi-day or a design change).\n- sharpened_recommendation: the most specific actionable fix in at most three sentences, naming files.\n\n\`\`\`json\n${JSON.stringify(f, null, 1)}\n\`\`\`` +} + +function criticPrompt(confirmed, minor, rejected, coverage) { + const line = (f) => `- [${f.id}] (${f.category}/${f.area}/${f.severity}) ${f.title} — ${f.code_ref}` + return `${PREAMBLE}\n\n## Task: completeness critic\nA review of this library ran ${finders.length} finders, each with an assignment:\n${finders.map(f => `- ${f.key} (${f.area}): ${f.prompt.slice(0, 220).replace(/\n/g, ' ')}...`).join('\n')}\n\nAfter adversarial verification there are ${confirmed.length} confirmed findings:\n${confirmed.map(line).join('\n')}\n\n${minor.length} confirmed-but-minor findings:\n${minor.map(line).join('\n')}\n\n${rejected.length} rejected findings (titles only):\n${rejected.map(f => `- ${f.title}`).join('\n')}\n\nFinder coverage notes:\n${coverage.map(c => `- ${c.key}: ${c.notes}`).join('\n')}\n\nAsk: what is missing? Which subsystems, files, doc sections or failure modes did no finder examine, or examined superficially (use the coverage notes)? Which confirmed findings hint at a larger class of issue that was not enumerated? Which cross-cutting questions are unaddressed (for example: Unicode handling end to end across JS, C++, Swift and Kotlin; long-session memory; RN version compatibility 0.82+ specifics; Expo integration; web platform; the TypeScript declaration output in dist; the release process; docs sections nobody audited)? Spot-check the repo yourself where it helps. Propose up to 8 targeted follow-up finder assignments, each with a concrete file list and specific questions, non-overlapping with what is already confirmed. If coverage is genuinely complete, return an empty followups list and say why.` +} + +function synthPrompt(confirmed) { + return `${PREAMBLE}\n\n## Task: synthesize verified findings into a prioritized structure\nInput: the confirmed findings (JSON) of a review of this library. Produce:\n- executive_summary: at most 220 words, plain and specific, no hype. Say what the library gets right, then the main problems, grouped by docs-vs-implementation, gaps, and design improvements.\n- headline_ids: the 8 to 12 finding ids in priority order, weighing severity, how directly it affects streaming performance or cross-block selection, and how badly the docs mislead.\n- themes: 4 to 8 themes, each with a 2 to 5 sentence narrative and member finding_ids. Every confirmed finding belongs to exactly one theme.\n- quick_wins: ids fixable in under an hour each.\n- docs_fix_list: every docs-vs-impl id.\n- scope_notes: anything the review could not establish (device behaviour, etc).\nRefer only to the given ids; invent nothing. Write plainly: short sentences, no em-dashes, no marketing language.\n\n\`\`\`json\n${JSON.stringify(confirmed.map(f => ({ id: f.id, title: f.title, category: f.category, area: f.area, severity: f.severity, effort: f.effort, doc_ref: f.doc_ref, code_ref: f.code_ref, claim: f.claim, reality: f.reality, recommendation: f.recommendation })), null, 1)}\n\`\`\`` +} + +// ---------------- verification of one finding ---------------- +async function verifyOne(f) { + const fact = await agent(factPrompt(f), { label: `fact:${f.id}`, phase: 'Verify', schema: FACT_SCHEMA, effort: 'high' }) + if (!fact) return { ...f, status: 'unverified', fact: null, mat: null } + if (!fact.confirmed) return { ...f, status: 'rejected', fact, mat: null } + const corrected = { + ...f, + title: fact.corrected_title || f.title, + code_ref: fact.corrected_code_ref || f.code_ref, + doc_ref: fact.corrected_doc_ref || f.doc_ref, + reality: fact.corrected_reality || f.reality, + } + const mat = await agent(matPrompt(corrected), { label: `impact:${f.id}`, phase: 'Verify', schema: MAT_SCHEMA }) + if (!mat) return { ...corrected, status: 'confirmed', fact, mat: null, effort: 'medium' } + return { + ...corrected, + status: mat.matters ? 'confirmed' : 'minor', + severity: mat.severity, + effort: mat.effort, + already_documented: mat.already_documented, + recommendation: mat.sharpened_recommendation || corrected.recommendation, + fact, + mat, + } +} + +function classify(list, confirmed, minor, rejected, unverified) { + for (const r of list) { + if (!r) continue + if (r.status === 'confirmed') confirmed.push(r) + else if (r.status === 'minor') minor.push(r) + else if (r.status === 'rejected') rejected.push(r) + else unverified.push(r) + } +} + +// ---------------- Phase 1: find ---------------- +phase('Find') +log(`Launching ${finders.length} finders`) +const raw = await parallel(finders.map(f => () => + agent(finderPrompt(f), { label: `find:${f.key}`, phase: 'Find', schema: FINDINGS_SCHEMA }) + .then(r => (r ? { ...r, key: f.key } : null)))) +const finderResults = raw.filter(Boolean) +const missingFinders = finders.filter((f, i) => !raw[i]).map(f => f.key) +if (missingFinders.length) log(`Finders that returned nothing: ${missingFinders.join(', ')}`) + +let nextId = 1 +const all = [] +for (const r of finderResults) for (const f of r.findings || []) all.push({ ...f, id: nextId++, finder: r.key }) +log(`${all.length} raw findings from ${finderResults.length}/${finders.length} finders`) + +const byArea = {} +for (const f of all) { + const a = AREAS.includes(f.area) ? f.area : 'architecture-design' + if (!byArea[a]) byArea[a] = [] + byArea[a].push(f) +} +const areaKeys = Object.keys(byArea).filter(a => byArea[a].length > 0) +log(`Areas: ${areaKeys.map(a => `${a}=${byArea[a].length}`).join(', ')}`) + +// ---------------- Phase 2+3: merge per area, then verify each merged finding ---------------- +const verifiedPerArea = await pipeline(areaKeys, + async (area) => { + const items = byArea[area] + if (items.length <= 1) return { area, merged: items.map(f => ({ ...f, id: `${area}-1`, source_ids: [f.id] })) } + const r = await agent(mergePrompt(area, items), { label: `merge:${area}`, phase: 'Merge', schema: MERGED_SCHEMA }) + if (!r || !r.merged) return { area, merged: items.map((f, i) => ({ ...f, id: `${area}-${i + 1}`, source_ids: [f.id] })) } + log(`merge:${area}: ${items.length} -> ${r.merged.length}`) + return { area, merged: r.merged.map((m, i) => ({ ...m, area, id: `${area}-${i + 1}` })) } + }, + async (m) => { + const results = await parallel(m.merged.map(f => () => verifyOne(f))) + return results.filter(Boolean) + }) + +const confirmed = [], minor = [], rejected = [], unverified = [] +for (const list of verifiedPerArea) if (list) classify(list, confirmed, minor, rejected, unverified) +log(`Round 0: ${confirmed.length} confirmed, ${minor.length} minor, ${rejected.length} rejected, ${unverified.length} unverified`) + +// ---------------- Phase 4: completeness critic, up to 2 follow-up rounds ---------------- +phase('Critic') +const coverage = finderResults.map(r => ({ key: r.key, notes: (r.coverage_notes || '').slice(0, 600) })) +const criticNotes = [] +let round = 0 +let critic = await agent(criticPrompt(confirmed, minor, rejected, coverage), { label: 'critic:1', phase: 'Critic', schema: CRITIC_SCHEMA }) +if (critic) criticNotes.push(critic.assessment) +let followups = critic && critic.followups ? critic.followups.slice(0, 8) : [] +while (followups.length && round < 2) { + round++ + log(`Critic round ${round}: ${followups.length} follow-up finders`) + const seenTitles = [...confirmed, ...minor, ...rejected].map(f => `- ${f.title}`).join('\n') + const extraRaw = await parallel(followups.map(f => () => + agent(finderPrompt({ key: f.key, area: f.area, prompt: f.prompt }, `\nAlready reported by earlier finders (do not repeat these; add only what is new):\n${seenTitles}\n`), { label: `find${round + 1}:${f.key}`, phase: 'Critic', schema: FINDINGS_SCHEMA }) + .then(r => (r ? { ...r, key: f.key } : null)))) + const extra = [] + for (const r of extraRaw.filter(Boolean)) { + coverage.push({ key: r.key, notes: (r.coverage_notes || '').slice(0, 600) }) + for (const f of r.findings || []) extra.push({ ...f, id: `r${round}-${nextId++}`, finder: r.key, source_ids: [] }) + } + log(`Critic round ${round}: ${extra.length} new candidate findings`) + const verifiedExtra = await parallel(extra.map(f => () => verifyOne(f))) + const before = confirmed.length + classify(verifiedExtra, confirmed, minor, rejected, unverified) + const gained = confirmed.length - before + log(`Critic round ${round}: +${gained} confirmed (${confirmed.length} total)`) + if (gained === 0 || round >= 2) break + critic = await agent(criticPrompt(confirmed, minor, rejected, coverage), { label: `critic:${round + 1}`, phase: 'Critic', schema: CRITIC_SCHEMA }) + if (critic) criticNotes.push(critic.assessment) + followups = critic && critic.followups ? critic.followups.slice(0, 6) : [] +} + +// ---------------- Phase 5: synthesize ---------------- +phase('Synthesize') +const synth = confirmed.length ? await agent(synthPrompt(confirmed), { label: 'synthesize', phase: 'Synthesize', schema: SYNTH_SCHEMA }) : null + +const strip = (f) => ({ + id: f.id, title: f.title, category: f.category, area: f.area, severity: f.severity, effort: f.effort, + already_documented: f.already_documented === true, doc_ref: f.doc_ref, code_ref: f.code_ref, + claim: f.claim, reality: f.reality, recommendation: f.recommendation, evidence: f.evidence, + finder: f.finder, source_ids: f.source_ids, fact_confidence: f.fact ? f.fact.confidence : null, + fact_reasoning: f.fact ? f.fact.reasoning : null, impact_rationale: f.mat ? f.mat.rationale : null, +}) + +return { + stats: { + finders: finders.length, findersReturned: finderResults.length, rawFindings: all.length, + confirmed: confirmed.length, minor: minor.length, rejected: rejected.length, unverified: unverified.length, + criticRounds: round, + }, + synthesis: synth, + confirmed: confirmed.map(strip), + minor: minor.map(strip), + rejected: rejected.map(f => ({ id: f.id, title: f.title, area: f.area, reason: f.fact ? f.fact.reasoning.slice(0, 400) : '' })), + unverified: unverified.map(f => ({ id: f.id, title: f.title })), + criticNotes, + coverage, +} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dc0536..1053377 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,12 +34,19 @@ jobs: # gate. While a second, pure-TypeScript parser existed, a runner with no # C++ toolchain could still prove the library and merely skip the native # measurements, so `--if-available` cost only numbers. Now a runner that - # skips this build runs *zero* markdown-parsing checks: jest quietly - # reports ten of twenty-six suites as skipped, the conformance run has - # nothing to score, and the job goes green having tested no parse. A - # missing compiler on ubuntu-latest is not a tolerable runner property, - # it is a broken job, so it fails here where the cause is named rather - # than passing silently downstream. + # skips this build runs *zero* markdown-parsing checks: every suite that + # parses markdown reaches the engine through the helpers in + # src/engine/native/__tests__/support.ts, and `describeNative` there is + # `describe.skip` without the addon. That is half the suite — + # + # grep -rlE 'describeNative|linkNativeEngineAsDefault|requireNativeEngine' \ + # src conformance --include='*.test.ts' --include='*.test.tsx' | wc -l + # + # returned 22 of the 45 suites under src/ and conformance/ as this was + # written — the conformance run has nothing to score, and the job goes + # green having tested no parse. A missing compiler on ubuntu-latest is + # not a tolerable runner property, it is a broken job, so it fails here + # where the cause is named rather than passing silently downstream. # # This must come BEFORE `npm test`: the src/engine/native/ suites load # build/selectable-markdown.node, and jest picks them up automatically. @@ -62,6 +69,63 @@ jobs: # copy-fidelity properties, and the tail-repair corpus. - run: npm test - run: npm run build + # Gate: adversarial input stays cheap in every stage a consumer runs, not + # only in the parse. md4c is linear on all four of these shapes, so a + # parse-only measurement gates the one stage that was never going to + # fail; the bench times tail repair, run segmentation and run projection + # beside it, and a throw (a stack overflow on deeply nested input, say) + # counts as a failure however fast it was. + # + # The budgets are PER STAGE because the stages differ by four orders of + # magnitude: measured medians on a developer machine are parse 23 ms, + # repair 31 ms, project 18 ms and segment 3.5 ms on the worst case of + # each, so one 1000 ms number — which is what this step used to pass — + # left `segment` free to get a hundred times slower and still pass. Each + # is 25-60x its measured median: not a performance target, a cliff + # detector, with enough headroom for a shared runner. The numbers + # themselves belong to docs/BENCHMARKS.md, measured on a machine whose + # clock is worth reading. + # + # `--require-engine` is what stops the step passing vacuously. Without + # it the bench exits 0 after printing that the addon did not resolve — + # a protocol-version drift between the built addon and dist/ is the + # likeliest way that happens here, and it would turn this gate green + # having measured nothing. + - run: >- + npm run bench:pathological -- + --require-engine + --budget-parse 750 --budget-repair 750 --budget-segment 200 --budget-project 750 + # Gate: adversarial STREAMING, which the document shapes above cannot + # see. `transcript-giant-list.json` is one 420-item bullet list, and a + # list never anchors — `StreamSession.isAnchorSafe` refuses it, a blank + # line does not end it — so every one of its 2484 appends re-reads the + # whole accumulated text and pays the splice on top. That is where a + # repair or splice pass that stops being linear shows up first, and it + # shows up in the number a user feels: `ms/chunk`. Before this step the + # bench had no budget, no non-zero exit and no workflow, so nothing + # anywhere failed on the adversarial streaming shape. + # + # Budgets are cliff detectors, not targets: measured on a developer + # machine the p99 append is 0.8 ms and finalize's single clean parse of + # the 22 kB result is 0.8 ms, so 20 ms and 50 ms leave 25-60x for a + # shared runner — the same headroom rule as the step above. `--repeat 1` + # because the p99 is taken over 2484 appends within one replay; the + # extra replays only steady the incremental-vs-full ratio, which is a + # published number rather than a gated one. `--require-engine` for the + # same reason it is on the step above. + - run: >- + npm run bench:streaming -- + --transcript conformance/fixtures/transcript-giant-list.json + --require-engine --repeat 1 + --budget-chunk 20 --budget-finalize 50 + # Gate: incremental projection still tracks the deltas rather than the + # document. This one is a COUNT, not a clock — source characters handed + # to `projectRun` over a whole replay — so it is deterministic and can be + # gated tightly: `cached` amplification must not grow more than 1.25x + # when the document doubles (it measures 0.99-1.00; the pipeline it + # replaced measures 1.98-2.09). It caught nothing before because nothing + # ran it: the bench printed the word "gate" and exited 0 regardless. + - run: npm run bench:projection -- --require-engine # Gate: the packed tarball is what a consumer actually installs, and # dist/ is never committed — this proves it still carries a working # entrypoint, the podspec, and the native sources. @@ -133,16 +197,59 @@ jobs: else sudo apt-get update && sudo apt-get install -y libboost-dev fi - # Gate: every .cpp this package ships to a consumer's compiler is - # compiled against the genuine React Native 0.75.4 renderer headers — on - # BOTH platform header sets on the macOS leg, on the Android set here on - # ubuntu — including the Android include-order seam that decides whether - # runs measure at all. + # Gate: every .cpp of the VIEW LAYER is compiled against the genuine + # React Native 0.75.4 renderer headers — on BOTH platform header sets on + # the macOS leg, on the Android set here on ubuntu — including the + # Android include-order seam that decides whether runs measure at all. - run: npm run check:fabric-cpp - # Gate: the checker can still fail. Five deliberately broken translation - # units must each be rejected, and rejected for the stated reason — - # against the iOS headers on macOS, the Android headers on ubuntu. + # Gate: the checker can still fail. Seven deliberate breakages must each + # be rejected, and rejected for the stated reason — against the iOS + # headers on macOS, the Android headers on ubuntu. Five of them + # (`MUTATIONS` in the script) are compiled into the synthetic Fabric + # translation unit the default run compiles; the other two + # (`SOURCE_MUTATIONS`) are applied to COPIES OF THE REAL + # platform/fabric sources, and they pin the clone-guard tripwire: that + # the static_assert fires when React Native moves both clean-clone + # mechanisms, and that the deliberately absent `override` is load-bearing + # rather than an oversight. - run: npm run check:fabric-cpp:selftest + # Gate: and every .cpp of the ENGINE, which the run above deliberately + # skips (it reaches no react/renderer header, and dragging the parser + # into the Fabric pass would only prove the harness works). Without this + # step platform/cpp compiled in no automated job at all — including + # SelectableMarkdownJsi.cpp, the installer that publishes + # `__selectableMarkdown` and therefore the only path the app has to the + # parser. It ships into libselectable-markdown.so through + # android/CMakeLists.txt and into the pod through the podspec's + # source_files, and nothing here runs Gradle or CocoaPods. + # + # `--syntax-only` at the podspec's own C++ standard is the whole check: + # the engine has no React Native dependency beyond , so a + # clean parse and semantic analysis is what a consumer's compiler is + # going to disagree with, and it takes seconds against the deps the run + # above already fetched. The platform is picked to match the runner + # because an explicit file list narrows check-fabric-cpp to one pass, and + # the iOS one needs an Apple SDK. + # + # The file list is built with `find`, not with `platform/cpp/*.cpp`. That + # glob is non-recursive and expanded by the runner's shell, so an engine + # source added in a subdirectory of platform/cpp would fall silently + # outside this gate — the default run reports such a file as skipped and + # nothing goes red. vendor/ is pruned because it is md4c's own C, built + # by scripts/build-node-addon.mjs at -std=c11 and not ours to hold to the + # podspec's C++ standard. An empty list fails the step rather than + # passing over nothing. + - name: Compile the markdown engine at the podspec's C++ standard + run: | + if [ "$RUNNER_OS" = "macOS" ]; then platform=ios; else platform=android; fi + files=$(find platform/cpp -path platform/cpp/vendor -prune -o -name '*.cpp' -print | sort) + if [ -z "$files" ]; then + echo "no .cpp found under platform/cpp — this gate would pass having compiled nothing" + exit 1 + fi + echo "$files" + # Unquoted on purpose: one argument per path. No path here has a space. + npm run check:fabric-cpp -- --syntax-only --platform "$platform" $files swift: name: Swift (real RN headers + iOS SDK) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8988311..5e89a5b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,24 +1,28 @@ name: Release # Tag-triggered only — `git tag v0.1.0 && git push origin v0.1.0`. Nothing -# here runs on an ordinary push or pull request. +# here runs on an ordinary push or pull request. `npm run release ` is +# the local half: it bumps, verifies and packs a dry-run tarball, then prints +# the commit/tag/push commands that get here. README.md's "Releasing" section +# is the same sequence in prose. on: push: tags: ['v*'] jobs: - release: + # ---- preflight: the checks that cost seconds ---------------------------- + # + # Its own job so a mistyped tag or a missing changelog section fails in under + # a minute instead of after the macOS gates below. Nothing here needs + # `npm ci`, which is why it does not run one. + preflight: + name: Preflight (tag, lockfile, changelog) runs-on: ubuntu-latest - permissions: - contents: write # attach the packed tarball to the GitHub release - id-token: write # npm publish --provenance steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 - cache: npm - registry-url: https://registry.npmjs.org # The podspec reads its version out of package.json, so a tag that # disagrees with it ships a pod CocoaPods cannot resolve. @@ -30,28 +34,155 @@ jobs: exit 1 fi + # `npm ci` installs from the lockfile and never compares the manifest's + # peerDependencies against it, so the lock's root entry kept claiming + # `react-native: >=0.73` through both 0.10.0 and 0.11.0 — a floor the + # package stopped supporting in 0.10.0 — with nothing anywhere going red. + # npm never publishes the lockfile, so the drift never reaches a + # consumer; it just makes the repository's own metadata lie. This is the + # only place it is checked. + # + # The comparison lives in a script rather than inline `node -e` because + # the inline version compared JSON.stringify output, which is key-ORDER + # sensitive: npm writes the lock's blocks alphabetically and a human + # appends to package.json, so an ordinary hand edit failed this step with + # a message showing two objects that read as identical. The script + # normalises key order and names the keys that actually differ, and being + # a file it can be run — and tested — outside a workflow. + - name: Check package-lock.json mirrors the manifest + run: node scripts/check-lock-sync.mjs + + # Release notes come from CHANGELOG.md. The same script runs again in the + # publish job, where its output becomes the notes file; here it is a + # gate, so a forgotten section costs seconds instead of a re-tag after a + # publish. `--generate-notes` would summarise the commits in the tag + # range, which on a squashed main is one commit message. + - name: Check CHANGELOG.md has a section for this tag + run: node scripts/changelog-section.mjs "$GITHUB_REF_NAME" > /dev/null + + # The other half of the same question. The step above proves the tag HAS + # notes; this one proves a breaking change is not shipping OUTSIDE them. + # Everything still sitting under `## [Unreleased]` when a tag is cut is in + # that tarball, so a line there saying BREAKING is a break published under + # a version whose notes never mention it — which is the state the last + # audit left behind (nineteen names dropped from the package root, an + # export renamed, a deep path that stopped resolving, all under an + # unbumped 0.11.0). The tag is passed explicitly because `actions/checkout` + # fetches the pushed ref and need not have any other tag locally. + - name: Check no BREAKING change ships under an already-tagged version + run: node scripts/check-unreleased-breaking.mjs --tag "$GITHUB_REF_NAME" + + # ---- the macOS gates ---------------------------------------------------- + # + # These used to be omitted from the release on the grounds that "CI runs both + # on macOS on every push to main and every pull request, which is where those + # regressions should be caught" — which assumes the tagged commit is one CI + # saw and saw pass. It is not: the trigger is `push: tags: ['v*']` with no + # branch constraint, so a tag can be pushed at any commit, on any branch, + # including one whose macOS legs were red. The omission rested on exactly the + # premise the comment below it rejects, and it meant a release could publish + # with the Swift selection host and the iOS renderer pass never compiled. + # + # So they run here too. The cost is real — this job is the long pole between + # pushing a tag and having an artifact — and it is the right trade for the + # last check before the npm registry. + macos-gates: + name: macOS gates (Swift + the iOS header set) + needs: preflight + runs-on: macos-latest + env: + # Turns the Swift script's "no Xcode, skipping" path into a failure. On + # this runner a skip would mean the image changed under us, and a + # silently skipped gate is worse than none — the same reasoning as + # ci.yml's swift job. + RNSM_REQUIRE_SWIFT: '1' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm - run: npm ci + # Boost is the only dependency check-fabric-cpp.mjs will not fetch for + # itself; everything else React Native pins is downloaded and cached by + # the script. + - run: brew install boost + # On macOS this compiles BOTH platform header sets — the iOS pass needs + # an Apple SDK and is the half the ubuntu job below cannot run. + - run: npm run check:fabric-cpp + - run: npm run check:fabric-cpp:selftest + # The engine, at the podspec's C++ standard. See the same step in ci.yml + # for why platform/cpp needs its own invocation, and why the file list is + # built with `find` rather than a non-recursive shell glob. + - name: Compile the markdown engine (iOS header set) + run: | + files=$(find platform/cpp -path platform/cpp/vendor -prune -o -name '*.cpp' -print | sort) + if [ -z "$files" ]; then + echo "no .cpp found under platform/cpp — this gate would pass having compiled nothing" + exit 1 + fi + echo "$files" + npm run check:fabric-cpp -- --syntax-only --platform ios $files + # The selection host itself: type-checked against React Native's real + # UIView (React) category, RCTViewManager and RCTUIManager, at the + # podspec's own deployment target and Swift version. + - run: npm run check:swift + - run: npm run check:swift:selftest + + release: + needs: macos-gates + runs-on: ubuntu-latest + permissions: + contents: write # attach the packed tarball to the GitHub release + id-token: write # npm publish --provenance + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + registry-url: https://registry.npmjs.org + + - run: npm ci + + # Preflight already proved this section exists; this run is the one whose + # output `gh release create --notes-file` reads. + # + # `set -o pipefail` is load-bearing: GitHub's default shell is + # `bash -e {0}`, where a pipeline's status is `tee`'s — always 0 — so a + # failure in changelog-section.mjs would leave an EMPTY notes file, pass + # this step, and publish a release with no notes. Preflight makes that + # unreachable in practice (same script, same checkout); it is one line to + # make the failure impossible rather than merely unlikely. + - name: Extract the release notes for this tag + run: | + set -o pipefail + node scripts/changelog-section.mjs "$GITHUB_REF_NAME" | tee "${RUNNER_TEMP:-/tmp}/release-notes.md" # THE SAME GATES CI RUNS, NOT A SUBSET. A tag can be pushed at any # commit — including one CI never saw, or one CI saw fail — and this # workflow is the last thing between the code and the npm registry. # Running only `npm test` here meant a release could ship a type error, # a codegen shape change that silently disables the component, or C++ - # that no longer compiles against React Native's headers. - # - # Everything below runs on this ubuntu runner. Two gates need macOS and - # are deliberately not repeated here, because adding a job the publish - # depends on doubles the time between pushing a tag and having an - # artifact: the Swift gate (Xcode), and the iOS half of check:fabric-cpp - # (React Native's iOS renderer headers need an Apple SDK, so on this - # runner the script skips that pass out loud and gates the Android - # pass). CI runs both on macOS on every push to main and every pull - # request, which is where those regressions should be caught. + # that no longer compiles against React Native's headers. The two gates + # that need macOS ran in the job this one depends on. - run: npm run typecheck - run: npm run check:codegen - run: sudo apt-get update && sudo apt-get install -y libboost-dev - run: npm run check:fabric-cpp - run: npm run check:fabric-cpp:selftest + # The engine, under Linux clang and libstdc++ — the Android-side reading + # of the same translation units the macOS job checked against the iOS + # header set. Same `find` enumeration as ci.yml, for the same reason. + - name: Compile the markdown engine (Android header set) + run: | + files=$(find platform/cpp -path platform/cpp/vendor -prune -o -name '*.cpp' -print | sort) + if [ -z "$files" ]; then + echo "no .cpp found under platform/cpp — this gate would pass having compiled nothing" + exit 1 + fi + echo "$files" + npm run check:fabric-cpp -- --syntax-only --platform android $files # HARD GATE, AND IT MUST COME BEFORE `npm test` — the same reasoning as # ci.yml. `describeNative` in src/engine/native/__tests__/support.ts is @@ -64,9 +195,25 @@ jobs: - run: node scripts/build-node-addon.mjs - run: npm test - # dist/ is never committed: `npm pack` runs prepare/prepack, which build - # it, and verify:pack proves the resulting tarball is consumable. + # dist/ is never committed: `npm pack` runs `prepare`, which builds it, + # and verify:pack proves the resulting tarball is consumable. - run: npm run verify:pack + # Gates: adversarial input stays cheap in every stage, a never-anchoring + # stream stays cheap per chunk, and incremental projection still tracks + # the deltas rather than the document. Same steps, same thresholds and + # same reasoning as ci.yml. + - run: >- + npm run bench:pathological -- + --require-engine + --budget-parse 750 --budget-repair 750 --budget-segment 200 --budget-project 750 + # And the adversarial STREAMING shape, on the never-anchoring transcript. + # Same step, same budgets and same reasoning as ci.yml. + - run: >- + npm run bench:streaming -- + --transcript conformance/fixtures/transcript-giant-list.json + --require-engine --repeat 1 + --budget-chunk 20 --budget-finalize 50 + - run: npm run bench:projection -- --require-engine - run: npm pack # The prebuilt tarball is the fast pin for consumers who cannot use the @@ -79,7 +226,8 @@ jobs: if gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then gh release upload "$GITHUB_REF_NAME" ./*.tgz --clobber else - gh release create "$GITHUB_REF_NAME" ./*.tgz --generate-notes --verify-tag + gh release create "$GITHUB_REF_NAME" ./*.tgz \ + --notes-file "${RUNNER_TEMP:-/tmp}/release-notes.md" --verify-tag fi # Skipped, not failed, when no NPM_TOKEN secret is configured — the diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..118eba4 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,338 @@ +# Changelog + +Notable changes per released version. Format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow +[semver](https://semver.org/), with the pre-1.0 rule that breaking changes +land in a **minor** bump. + +`main` was squashed to a single commit at the 0.10.0 tag, and GitHub release +notes are generated from the commits in a tag range — which for 0.10.0 is one +commit called "Initial commit". That is why this file exists: it, not the +release notes, is the record of what changed. `.github/workflows/release.yml` +reads the section for the tag being released and refuses to publish without +one. + +Entries before 0.10.0 are not reconstructed here; the per-release detail for +those exists only in the pre-squash branches on the remote. + +## [Unreleased] + +An audit pass across the whole library. Two entries under Changed are +breaking; under the pre-1.0 rule they land in the next **minor** (0.12.0). +Nothing here is published yet: `package.json` is still at 0.11.0, so the next +publish has to bump the version and move this section under that heading — the +release workflow reads the section for the tag it is given and refuses to +publish without one. + +### Added + +- **Streaming: `repairTail` resumes its inline scan.** It takes an optional + carry-forward `RepairScan` — returned as `RepairResult.scan` and exported + from the package entry — so successive calls on a growing tail pick up where + the last one stopped instead of re-deriving the inline region from the start + of the tail. Passing nothing behaves exactly as before, and the function is + still pure. `RepairResult.scan` is optional, so code that builds a + `RepairResult` literal still compiles; omitting it only costs the resume. +- **Selection: an imperative API.** `` forwards a ref + (`SelectableMarkdownHandle`) with `getSelection()`, `clearSelection()` and + `setSelection(span)`, and takes an `onSelectionChange` prop that reports the + live selection as `{ span, plain }`, or `null` when it goes away. Passing a + `ref` used to be a type error, so an app could not build its own floating + toolbar, clear a stale selection on navigation, or highlight a span it had + computed. `setSelection` returns `boolean`: false when no mounted run both + shows the span and will take a selection right now. Scrolling a span into + view is not part of it: the document owns no scroll view, and the wire has + no measurement channel yet. +- **View: `RunHostHandle`.** `RunHost` forwards a ref with `clearSelection()` + and `setSelection(start, end)` in the run's own display offsets, and + `mapSourceToRunRange` (with the `RunTextRange` type) is exported so a + consumer driving runs themselves can do the mapping `` + does internally. Both hosts implement the matching `onSelectionChange` event + and `clearSelection`/`setSelection` codegen commands. + `RunHostHandle.setSelection` returns `boolean` — true when the command was + dispatched, false when nothing was asked of the platform — which anyone + implementing the interface has to return. +- **View: ``.** The run cap is a prop now, not + only an option on direct `segmentRuns` calls: it forwards straight through, + defaults to `DEFAULT_MAX_RUN_CHARS` (8000), and takes `Infinity` to opt out. +- **View: `RenderContext.selectable`.** Optional, and ABSENT MEANS + SELECTABLE: every built-in renderer, including the unknown-kind and + depth-cap fallbacks, reads it as `ctx.selectable ?? true`, so a hand-built + context still compiles and still renders selectable text. The view sets it + from the platform tail policy. +- **Selection: `CopyContext.classifyBlock`,** so a copy re-segments the + reparsed slice the way the document was segmented. +- **Selection: `exclusiveSelection`** (default `true`) on + `` and `RunHost`. `false` opts a document's runs out of + the process-wide one-active-selection coordination in both directions, so + two selections can be held at once and a cross-message copy is reachable by + hand. +- **Selection: `ProjectedRun.extents`** and the `ProjectedExtent` type, which + carry each construct's projected range alongside its own source span. +- **Selection: `segmentRuns` takes `maxRunChars`**, with the exported constant + `DEFAULT_MAX_RUN_CHARS` (8000). +- **View: incremental run projection for consumers.** + `createRunProjectionCache` and the `RunProjectionCache` type, plus + `ProjectRunOptions.previous` and the `PreviousProjection` type on + `projectRun`, so a consumer driving `RunHost` gets the same reprojection + `` uses instead of reprojecting a settled run from the + start on every delta. +- **View: an `images` prop** (`'embed' | 'standalone'`, default `'embed'`), + the `withImageEmbeds` helper and the `ImageMode` type, and a new theme token + `spacing.imageWidth` (default 280). +- **View: selection-menu titles.** A `selectionActions` entry may now be + `{ id, title }`, so labels come from the app's own i18n on both platforms, + and consumer-defined ids work end to end: rendered when they carry a title, + routed back through `onSelectionCopy` with that id and the usual payload. + `handleSelectionAction` gained `ctx.actions`, which is a DEV cross-check + only: it warns once per id when an action arrives that the offered menu did + not contain, and never renames it. +- **View: three DEV warnings.** One for a `selectionActions` id that is neither + built-in nor titled, which both hosts drop rather than render blank — latched + per id rather than once per JS runtime, so a second document in a transcript + offering a different untitled id warns again. One for an action id reported + back that `ctx.actions` never offered. One for + `urlPolicy.blockedLinks: 'node'` paired with a `link` renderer override and + none of `onLinkPress`, `embed` or `classifyBlock` — the one shape where the + override never runs, because flowing prose has no renderers in it. +- **Engine: `sanitizeUrl` and `isUrlAllowed`** are exported from the package + entry, so an engine author applies the same allowlist the view re-checks + with rather than reimplementing it. `isNativeEnginePermanentlyRefused()` is + exported next to `isNativeEngineAvailable()`. +- **Engine: semantics on the run wire.** `RunTextAttribute` and + `NativeRunTextAttribute` gained six fields — `role` + (`'heading' | 'listItem' | 'tableCell'`), `roleLevel`, `roleRow`, + `roleRowCount`, `roleColumn`, `roleColumnCount` — a channel separate from + styling. None of them is reachable from `attributeForMark`, which returns + `RunMarkStyle`. +- **iOS and Android: headings, list items and table cells inside a merged run + are announced.** VoiceOver and TalkBack reach headings as headings and + navigate heading to heading, instead of hearing prose in a larger font, and + each list item and table cell is its own focus stop — on Android a virtual + view carrying `CollectionItemInfoCompat`, so TalkBack says "row 2, column 3" + in the reader's own language. The roles come from the semantic fields on the + wire, so a restyled heading is still a heading; Android's font-shape + inference is gone. Code-block and blockquote structure still does not cross: + neither platform has a primitive for it. +- **Android: `res/values/strings.xml`** ships + `selectable_markdown_copy_text` and `selectable_markdown_copy_markdown`, so + a host app rewords or translates the default menu items through ordinary + Android resources. +- **Packaging: an `exports` map.** The root, `./dist`, `./dist/*`, `./src/*`, + `./package.json` and `./react-native.config.js` are what a consumer can + import, and every `dist` entry carries `require` and `import` conditions + with their own `types`; the package is marked `sideEffects: false`, object + files are excluded from the tarball, and `CHANGELOG.md` ships in it. +- **Packaging: an ES module build.** `npm run build` also emits `dist/esm` + (`tsconfig.esm.json`, `module: es2020`), finished by + `scripts/finish-esm-build.mjs` — which writes `dist/esm/package.json` + (`"type": "module"`, `sideEffects: false`) and adds the `.js` extension + Node's ESM resolver needs on every relative specifier. `package.json` gains + a `module` field and the `import`/`require` conditions, with `react-native` + still first and still `src/index.ts`; `npm run verify:pack` resolves and + imports the documented deep paths under both conditions out of the tarball. + Bundler-visible, not breaking — the `require` path is byte-identical to what + shipped, and the ESM build is what lets webpack, Rollup and Vite tree-shake + per export. +- **Packaging: `npm run build` emits `dist/view/SelectableRunHostNativeComponent.d.ts`** + beside the shim it already emitted, so the `./dist/*` types condition + (`./dist/*.d.ts`) has no hole; `npm run verify:pack` fails the tarball when + the declaration is missing. +- **Benches and CI: gates that cannot pass vacuously.** `bench:pathological` + takes per-stage budgets (`--budget-parse|-repair|-segment|-project MS`) and + `--require-engine`, which turns an unresolvable addon from an exit-0 report + into a failure; `bench:projection` is a gate CI runs, failing when cached + projection amplification grows more than 1.25× across a document doubling; + and `scripts/check-lock-sync.mjs` runs in release.yml's preflight, comparing + the ten fields npm actually copies into the lock's root entry — name, + version, license, the five dependency blocks, `engines` and `bin` — + key-order-insensitively, and treating an empty block and a missing one as + the same statement, so `"dependencies": {}` against a lock that omits the + block is no longer a red describing no drift. +- **Docs: release notes come from this file.** `scripts/changelog-section.mjs` + prints one version's section for `gh release create --notes-file`, and the + release workflow fails in its first job when the section for the tag is + missing, instead of generating notes from a squashed commit log. +- **Release guard: a pending breaking change cannot ship under a version that + is already tagged.** `scripts/check-unreleased-breaking.mjs` fails when this + file's Unreleased section marks a change breaking while `package.json`'s + version equals the version of the latest `v*` tag. It runs in release.yml's + preflight job (passed the tag being pushed, so a shallow checkout still has + something to compare against) and in `npm run release` right after the + version bump, where a failure rolls the bump back. Against this tree as it + stands the guard exits 1, which is the intended state: everything under + Unreleased ships in the tarball a tag produces, so the two breaking entries + below have to move under the new version's heading and the version has to be + bumped past v0.11.0 — 0.12.0 under the pre-1.0 rule — or the tag fails + preflight in seconds. No version was bumped by this change. (This entry + spells the marker in lower case on purpose: the guard matches the literal + upper-case word, line by line.) + +### Changed + +- **BREAKING — the package entry publishes an explicit list** of 190 names + instead of two dozen `export *` lines. Nineteen internals are no longer at + the root and are imported by path instead: `__linkNativeEngine`, + `decodeFlatBuffer`, `NativeProtocolError`, `applySmartPunctuation`, + `PROTOCOL_VERSION`, `findHostBinding` and `NativeHostBinding` from + `dist/engine/native`; `encodeSelectionActions`, `decodeSelectionAction`, + `SELECTION_ACTION_SEPARATOR`, `isBuiltInSelectionAction`, + `selectionActionId` and `selectionActionTitle` from + `dist/view/selectionActions`; `getOrCreateSession`, `resolveSessionInit`, + `settleUnboundSession` and `useDeferredUnmount` from + `dist/agui/useAgUiSession`; `embedContentFor` from `dist/selection/runs`; + `isUriLikeLabel` from `dist/stream/repair`. Nothing was made private, but a + deep path carries no stability promise. +- **BREAKING — `classifyBlock`, the exported function, is now + `classifyTopLevelBlock`.** The `classifyBlock` prop of + `` and the `ClassifyBlock` type are unchanged; the + rename exists because all three sat at the package root under two spellings + of one word. +- **Streaming: a release never cuts inside a grapheme cluster.** The + smoother's budgeted cut moves up to the next cluster boundary and + `holdBackChars` moves down to the previous one, so a flag, an emoji ZWJ + sequence or a combining mark is never painted as half a glyph. + `Intl.Segmenter` is not used; Hermes ships a limited Intl subset. +- **Selection: prose merging stops at `DEFAULT_MAX_RUN_CHARS`** source + characters, so a document longer than that renders as several native hosts + rather than one and a sweep cannot cross the boundary. It breaks merging + between blocks and never splits one, so a single block past the cap is still + one run. Pass `maxRunChars: Infinity` — as the `segmentRuns` option or the + `` prop — for the previous behaviour. `classifyBlock` results + are memoized on block identity, which makes the callback's documented purity + and referential stability load-bearing. +- **View: an image no longer makes its containing block standalone.** It is + claimed as an embed and flows inside its run, so a sweep across an + illustrated answer stays whole and the block keeps `onSelectionCopy`. A + `classifyBlock` claim on an image node no longer forces standalone either; + `images: 'standalone'` is what does that now. +- **View: `openUrl(href, allowedPrefixes?)` sanitizes and re-checks** the href + against `urlPolicy.linkPrefixes` before `Linking.openURL`, and opens the + sanitized string, so a href from a substituted engine cannot bypass the + allowlist. A caller passing no prefixes gets `DEFAULT_LINK_PREFIXES`; + `RenderContext` gained `linkPrefixes`. +- **View: `SelectionCopyEvent.action` and `SelectionActionEvent.action`** + widen from the closed `'copy-text' | 'copy-markdown'` union to + `SelectionActionId`, and every non-empty id is now reported VERBATIM, + built-in or consumer-defined, whether or not the caller threaded + `ctx.actions`. Only an event carrying no action at all resolves to + `'copy-markdown'` — that is the whole of the version skew, a binary older + than the `action` field. Renaming an id the offered list did not contain + could only hand a consumer their own action in their copy-markdown branch. + Not breaking otherwise: a bare string id behaves exactly as before, and the + default menu's wire bytes are unchanged. +- **View: the `selectionActions` memo compares every field.** The entry-by-entry + comparison behind the inline form is now a generic shallow compare of every + own field on each entry, in both directions, rather than `id` and `title` by + name, so a field added to `SelectionActionSpec` later is not silently + invisible to the memo. A bare string id still compares equal to a spec that + adds nothing beyond that id, because both encode to the same wire entry. +- **View: `MarkAttribute` returns the narrower `RunMarkStyle`** — + `RunTextAttribute` without `start`, `end` and the six semantic fields. Any + existing `attributeForMark` still compiles. +- **View: `colors.quoteBar` ships no value.** It is gone from `defaultTheme` + and `defaultDarkTheme` (it was `'#c9ced6'` / `'#3d444d'`) because nothing + read it: `quote.barColor` is what renders, so overriding that alone left + `colors.quoteBar` reporting the old colour. It still works as a deprecated + input — setting it alone recolours the bar — but code that reads the theme + for its own drawing must read `theme.quote.barColor`. +- **Engine: `NativeModules.SelectableMarkdown.install()` returns + `'installed' | 'unavailable' | 'refused'`** instead of a boolean. Both + directions stay compatible: an older JS bundle ignores the return value and + reads the global, and a newer bundle meeting an older binary reads `false` + as the transient `unavailable`. The point is that `isNativeEngineAvailable()` + no longer re-crosses the bridge on a permanent refusal, so it is safe to + poll from a render path. + +### Fixed + +- **Selection: copy-markdown no longer drops a block's own syntax.** A + selection covering a whole heading, list, list item, blockquote, fenced or + indented code block, table, emphasis or strong span, or inline link now + carries that construct's source, so a copied list pastes as a list instead + of re-parsing as a paragraph. A partly covered construct is unchanged. +- **Selection: an escape, entity, smart quote or ellipsis no longer pins its + whole text node.** In plain prose the decoder emits one text node per + paragraph, so a single `\*` or `…` used to make a twelve-character + selection copy the entire paragraph. Such a node is now covered piecewise + and only the respelled stretch is indivisible. +- **Selection: an indented code block mapped one indent-width to the left.** + The block's literal is de-indented and newline-terminated while its source + slice is neither, and at some widths the two come out the same length, which + the piece table read as a character-for-character mapping. Copying from an + indented code block now returns the source it displays. +- **View: an embed claim with an infinite `width` or `height` is rejected.** + `Infinity > 0` passed the old `!(x > 0)` gate and reached the hosts as an + infinite `CGRect` and an `Int.MAX_VALUE` span. +- **iOS: a blockquote bar no longer lands on top of right-to-left text.** Head + and tail indents are LEADING-edge relative, and TextKit resolved that edge + per paragraph from its own first strong character, so a single Arabic or + Hebrew quote inside an English transcript indented from the right while the + bar was painted at the physical left. The attributed-string builder now pins + `baseWritingDirection` on every range it indents — to the direction TextKit + would have resolved anyway, so nothing that renders today moves — and + `draw(_:)` reads that decision back off the string to put the bar on the + leading edge. Measurement and painting can no longer disagree, because only + one of them chooses. +- **Android: the selection menu was hardcoded English** with no override path. + +## [0.11.0] — 2026-09-01 + +### Added + +- **Embeds, restored and Fabric-only** (removed in 0.10.0). `` + takes an `EmbedRenderer`: a claimed node keeps flowing through its selection + run as a single placeholder character mapped to the node's whole source + span, the native host reserves the declared `width` × `height` there, and + `onEmbedLayout` reports where the reservation landed so the element can be + overlaid on it. One sweep still selects across an answer containing a card, + and copying it yields the node's exact markdown (`copy-markdown`) or the + declared `text` (`copy-text`). +- `RunEmbed` and the embed projection helpers are public again from the + package entry, alongside the `embeds` prop and `onEmbedLayout` event on + `RunHost`. +- The codegen spec carries `embeds` and `onEmbedLayout`, and + `npm run check:codegen` asserts both survive codegen on iOS and Android. +- Native support on both platforms: placeholder-aware attributed text and + rect reporting in the iOS host, `RunEmbeds` / `EmbedLayoutEvent` and a + layout cache that keys on embeds in the Android host. +- Selection tests grew with it: the projection oracle, run segmentation and + copy-fidelity suites all cover embedded runs. + +## [0.10.0] — 2026-09-01 + +The release that dropped the old architecture. Every removal below is +breaking. + +### Changed + +- **Peer floor raised to `react-native >= 0.82`** (from `>= 0.73`). From 0.82 + React Native refuses an old-architecture `pod install` outright, so the + package targets bridgeless Fabric only. +- `` renders the session's snapshot and no + longer re-parses with the component's `options`/`engine`: the session owns + its parse context. +- The published tarball no longer contains tests — `files` excludes + `src/**/*.test.ts(x)` and `src/**/__tests__`. + +### Removed + +- **The old-architecture (Paper) selection host.** The Kotlin measuring shadow + node, the Swift and Objective-C view managers, and the podspec's + architecture gate are gone; the podspec now adds the Fabric sources and + calls `install_modules_dependencies` unconditionally. +- **The `` fallback.** `RunHost` throws where the native + component is not registered (Expo Go, web, a test renderer, a binary built + without the pod) instead of rendering a document that merely looks + selectable — on iOS Fabric that tier was a long-press whole-block Copy menu + with no handles, no range and no `onSelectionAction`, and two defects hid + behind the appearance for a release each. The error names the missing + `RCTThirdPartyComponentsProvider` entry and the rebuild it needs. +- **Embeds.** The `embed` prop, `RunEmbed` and the native reservation path + were removed as unused; restored in 0.11.0. +- `StreamSessionInit.onUpdate`. Use `session.subscribe(listener)`, which + returns an unsubscribe function. + +[Unreleased]: https://github.com/superpowerdotcom/react-native-selectable-markdown/compare/v0.11.0...HEAD +[0.11.0]: https://github.com/superpowerdotcom/react-native-selectable-markdown/releases/tag/v0.11.0 +[0.10.0]: https://github.com/superpowerdotcom/react-native-selectable-markdown/releases/tag/v0.10.0 diff --git a/README.md b/README.md index ec5a53c..b6bed47 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # react-native-selectable-markdown -Markdown rendering for React Native, built for streamed LLM output, with selection that works like a document. Adjacent paragraphs, headings, lists, code blocks and tables merge into one selectable run, and every selected character maps back to an exact UTF-16 range of the source. Copy gives you the markdown itself. +Markdown rendering for React Native, built for streamed LLM output, with selection that works like a document. Adjacent paragraphs, headings, lists, code blocks and tables merge into one selectable run, and every selected character maps back to an exact UTF-16 range of the source. Copy gives you the markdown itself: the exact source slice, syntax included for any construct the selection covers in full. Parsing is done by [md4c](https://github.com/mity/md4c), compiled into your app. There is no JavaScript parser, so nothing renders until the app is rebuilt with the native module linked (see [Native setup](#native-setup)). You can swap in your own parser through the `Engine` interface. MIT licensed. @@ -14,8 +14,9 @@ The fix for all three is a `SourceSpan` on every parsed node. Selection, copy, m ## Features -- Selection across adjacent blocks. Code blocks, tables and rules flow through runs too, with their boxes and rules painted as decorations under the text. A run ends at an image, a spoiler, or a block you mark standalone. -- Copy Text and Copy Markdown in the selection menu. The second is an exact source slice. +- Selection across adjacent blocks. Code blocks, tables and rules flow through runs too, with their boxes and rules painted as decorations under the text. Images flow as embeds: the host reserves space on the line and the image renderer draws over it. A run ends at a whole block — the list or the table, not just the construct — that holds a spoiler, an image nothing claimed, or a node you mark standalone. Merging also stops at `maxRunChars` (a prop; 8000 source characters by default, `Infinity` opts out), which only ever breaks between top-level blocks. +- Copy Text and Copy Markdown in the selection menu. The second is an exact source slice. Both are retitleable from JS, and your own menu items report back the same way. +- An imperative selection API. `onSelectionChange` reports the live selection as an exact source span, including when it goes away, and a ref gives you `getSelection()`, `clearSelection()` and `setSelection(span)`. - Streaming without artifacts. The unsettled tail is repaired before parsing, settled blocks keep referential identity, and plain-prose deltas skip the parser. Per-frame coalescing and a typewriter smoother are opt-in. - Safe defaults for model output. Every non-CommonMark extension is opt-in, HTML is stripped, URL schemes are allowlisted. - 651/652 on the CommonMark 0.31.2 spec suite, run in CI, plus a streaming oracle that checks every prefix of every fixture against a fresh parse. @@ -27,7 +28,7 @@ The fix for all three is a `SourceSpan` on every parsed node. Selection, copy, m npm install react-native-selectable-markdown ``` -Other ways in: a pinned commit (`npm install github:superpowerdotcom/react-native-selectable-markdown#`, rebuilds on every install), a tarball from the [releases page](https://github.com/superpowerdotcom/react-native-selectable-markdown/releases) (prebuilt), or a local checkout (`npm install file:../react-native-selectable-markdown`; `file:` deps are symlinked, so run `npm run build -- --watch` in the checkout yourself). `dist/` is never committed; `prepare` builds it for the npm and git paths. +Other ways in: a pinned commit (`npm install github:superpowerdotcom/react-native-selectable-markdown#`, rebuilds on every install), a tarball from the [releases page](https://github.com/superpowerdotcom/react-native-selectable-markdown/releases) (prebuilt), or a local checkout (`npm install file:../react-native-selectable-markdown`; `file:` deps are symlinked, so run `npm run build:watch` in the checkout yourself — Metro takes the `react-native` condition straight to `src/`, so the watch build only matters for bundlers that resolve `main`). `dist/` is never committed; `prepare` builds it for the npm and git paths. What changed per version is in [CHANGELOG.md](CHANGELOG.md). ### Native setup @@ -45,9 +46,9 @@ if (!installNativeEngine()) { } ``` -Without the module, `parseDocument` throws on the first non-empty document, and so does `` during render; wrap it in an error boundary if that is a state your app can be in. `RunHost`, the selection host, throws where its Fabric component is not registered (Expo Go, web, test renderers). It ships as a Fabric component only, matching the `react-native >= 0.82` peer floor. `use_frameworks! :linkage => :dynamic` disables registration of all third-party Fabric components with no error, this one included. +Without the module, `parseDocument` throws on the first non-empty document, and so does `` during render; wrap it in an error boundary if that is a state your app can be in. `RunHost`, the selection host, throws where its Fabric component is not registered (Expo Go, web, test renderers). It ships as a Fabric component only, matching the `react-native >= 0.82` peer floor. Registration comes from this package's `codegenConfig.ios.componentProvider`: it puts a `SelectableRunHost` entry in the app's generated `RCTThirdPartyComponentsProvider.mm`, and the component is looked up by name from there — which is what the throw message tells you to check. -Where native code cannot run at all, everything above the parser is plain TypeScript and still works. Pass your own `Engine`; its one hard requirement is a `SourceSpan` on every node, with UTF-16 offsets into exactly the source it was handed. +Where native code cannot run at all, everything above the parser is plain TypeScript and still works. Pass your own `Engine`; its one hard requirement is a `SourceSpan` on every node, with UTF-16 offsets into exactly the source it was handed. Streaming is the one layer that assumes CommonMark on top of that: tail repair appends virtual closers and suppresses ambiguous tail lines before the engine sees them, and the parse-free fast path keys on CommonMark's construct characters, so a delta that only opens syntax of your own renders as stale text until the next construct character arrives. ## Quick start @@ -71,7 +72,9 @@ export function Message({ markdown }: { markdown: string }) { } ``` -`selectionActions` and `onSelectionCopy` work as a pair. Setting one without the other yields an empty custom menu. +`selectionActions` needs `onSelectionCopy`: with no handler the custom menu is empty rather than one item short, because an item that reports nowhere is not offered (DEV warns). The reverse is not true — a handler on its own is the common case and gets both default items. An entry may also be `{ id, title }`, which is the one way to localise both platforms from JS; any id beyond the two built-ins is your own action and must carry a title, and it arrives at `onSelectionCopy` with the same `plain`, `markdown` and `span`. A `session` supersedes `source`, which is then never parsed (DEV warns). + +For a toolbar of your own, `onSelectionChange` reports `{ span, plain }` as the user drags and `null` when the selection goes away — the half `onSelectionCopy` cannot give you, since it fires only after a menu item is tapped. A `ref` typed `SelectableMarkdownHandle` adds `getSelection()`, `clearSelection()` and `setSelection(span)`. `setSelection` returns `false` for two kinds of refusal: no run shows the span (past the end of the document, a standalone block, a span of pure markup like a fence or a `# `, a run not mounted yet), or the run that shows it cannot take a selection right now (Android's unsettled streaming tail, a run rendered `selectable={false}`, a binary built against a native spec older than the selection commands). It can select wider than asked when the span lands inside an entity, an image's alt text or an embed placeholder, and it presents no menu and issues no scroll — there is no scroll-to-span, though it does take focus, which a scrolling ancestor is entitled to react to. Full contract in [docs/SELECTION.md](docs/SELECTION.md). ### Streaming @@ -86,7 +89,7 @@ session.finalize('end'); // or 'aborted' | 'failed'; idempotent ; ``` -`append` parses synchronously. For tokens that arrive faster than frames, `appendBuffered(delta)` coalesces them into one flush per frame. `flushBuffered()` drains now, and any synchronous call (`append`, `replace`, `finalize`) drains first. +`append` parses synchronously. For tokens that arrive faster than frames, `appendBuffered(delta)` coalesces them into one flush per frame. `flushBuffered()` drains now, and any synchronous call (`append`, `replace`, `finalize`) drains first. `dispose()` cancels the scheduled flush and the idle drain, drops the un-appended tail and clears subscribers — call it for a session dropped before its stream ends, after `flushBuffered()` if the tail should be kept. ```ts new StreamSession({ @@ -98,7 +101,7 @@ new StreamSession({ }); ``` -A `Smoother` decides how many UTF-16 units each flush releases, and the session keeps flushing until the buffer drains. `createSmoother({ charsPerSecond, boundary, maxLagChars })` is a fixed rate. `createAdaptiveSmoother()` is what we use for live LLM streams: it tracks the arrival rate, trails the head by a target lag, and drains against a bounded deadline at run end (call `session.notifyRunFinalized()`; the ag-ui binding does this for you). Await `session.drained()` before `finalize` so the tail finishes typing. `session.rewrite(full)` edits text the reader has not seen yet without interrupting the reveal. Details in [docs/STREAMING.md](docs/STREAMING.md). +A `Smoother` decides how many UTF-16 units each flush releases, and the session keeps flushing until the buffer drains. Only scheduled `appendBuffered` flushes are smoothed: `append`, `flushBuffered`, `replace` and `finalize` drain synchronously and release everything, so a session driven by `append` alone never consults the smoother and `drained()` resolves immediately. `createSmoother({ charsPerSecond, boundary, maxLagChars })` is a fixed rate; `boundary: 'word'` needs whitespace to cut at, so text written without spaces (Chinese, Japanese, Thai) wants `boundary: 'char'`. `createAdaptiveSmoother()` is what we use for live LLM streams: it tracks the arrival rate, trails the head by a target lag, and drains against a bounded deadline at run end (call `session.notifyRunFinalized()`, a no-op with nothing pending; `bindRunTextEvents` and `useAgUiRunSessions` do it for you). Await `session.drained()` before `finalize` so the tail finishes typing, and handle its rejection: a buffered drain the engine keeps refusing runs out its retry ladder (eight retries, about 20 s at the default `holdIdleMs`) and rejects every waiter with the engine's own error. Nothing is dropped — the text stays buffered, `pendingLength` still counts it, and an explicit `flushBuffered`, `append`, `replace` or `finalize` retries it. `session.rewrite(full)` edits text the reader has not seen yet without interrupting the reveal. Details in [docs/STREAMING.md](docs/STREAMING.md). ```ts import { StreamSession, createSmoother } from 'react-native-selectable-markdown'; @@ -106,9 +109,9 @@ import { StreamSession, createSmoother } from 'react-native-selectable-markdown' const session = new StreamSession({ smoother: createSmoother({ charsPerSecond: 300, boundary: 'word', maxLagChars: 400 }), }); -// ...append tokens... -await session.drained(); -session.finalize(); +for (const token of tokens) session.appendBuffered(token); // buffered, not append +await session.drained().catch(() => {}); // rejects if the engine gave up on the buffered tail +session.finalize(); // an explicit drain retries the held text ``` ### ag-ui @@ -125,7 +128,9 @@ function AssistantMessage(props: { events: TextMessageEvents; messageId: string } ``` -The session finalizes on message end and on run finished or failed, since an aborted stream never sends END. For a transport that owns a whole run, `bindRunTextEvents(events, store, policy?)` and its hook `useAgUiRunSessions(events, init?)` manage per-message sessions: they seed pre-existing messages without re-typing, route new ones through `appendBuffered`, and report `holding: true` until every smoother has drained at run end. Policy fields are documented on `RunBindingPolicy`. +The session finalizes on message end and on run finished or failed, since an aborted stream never sends END; a `messageId` switch settles the outgoing session as `'aborted'` rather than stranding it mid-stream, and it stays in the hook's map, so switching back shows the settled document. The third argument is either bare parse options, as above, or a full session init — `{ engine, options, coalesce, smoother, holdBackChars, holdIdleMs, repair, bufferScheduler, idleScheduler, now }`, where `smoother` is a factory because there is one session per messageId. Each delta commits on its own unless the init carries a buffering field (`smoother`, `holdBackChars`, `holdIdleMs`, either scheduler) or an explicit `coalesce: true`; `repair` and `now` deliberately do not switch coalescing on. Every field is latched when a messageId's session is created, so changing the init later reaches the next new message, not the one already streaming. What the per-message adapter does not do is hold at run end: neither `useAgUiSession` nor `bindMessageEvents` calls `notifyRunFinalized`, so run end finalizes straight through a metered tail — a smoothed reveal commits the rest in one revision instead of playing it out. + +For a transport that owns a whole run, `bindRunTextEvents(events, store, policy?)` and its hook `useAgUiRunSessions(events, init?)` (the same session fields, plus `policy`) manage per-message sessions: they seed pre-existing messages without re-typing, route new ones through `appendBuffered`, and report `holding: true` until every smoother has drained at run end. Policy fields are documented on `RunBindingPolicy`. The three run-lifecycle callbacks take an optional trailing `runId`; pass it when the transport has one, and `bindRunTextEvents` ignores a late run finished or failed from a run it already watched go spent. An id stops being spent the moment that run starts again — announce a retry that reuses it with `onRunStarted` and it keeps its right to finalize — and `onAttached` catch-up clears the spent-run memory outright, since nothing learned before a gap the binding cannot see into is trustworthy. `bindMessageEvents` observes no run start and filters nothing. ### Headless @@ -136,7 +141,7 @@ const doc = parseDocument('# Hello *world*', presets.llmChat); visit(doc, (node) => { /* node.span = { start, end } into doc.source */ }); ``` -In plain Node, import deep paths (`dist/engine/Engine`, `dist/stream/StreamSession`, ...), since the package root re-exports `react-native`. Parsing there still needs an engine; md4c is native. See [docs/NATIVE.md](docs/NATIVE.md). +In plain Node, import deep paths (`dist/engine/Engine`, `dist/stream/StreamSession`, ...), since the package root re-exports the React Native view layer, which imports `react-native` at load time. Those paths are declared in the `exports` map, which serves two builds: `require` takes the CommonJS tree in `dist/`, `import` takes an ES module build in `dist/esm`, and Metro's `react-native` condition still wins and still points at `src/index.ts`. The ESM build is what lets webpack, Rollup and Vite tree-shake per export rather than only drop a module nothing imports (`sideEffects: false` is declared in both package.json files, because bundlers read it from the nearest one). Types resolve under both conditions — `dist/*.d.ts` for `require`, `dist/esm/*.d.ts` for `import` — `react-native-selectable-markdown/dist`, the bare directory, resolves under both, and `npm run verify:pack` resolves *and* imports each documented deep path through both conditions out of the packed tarball. Parsing there still needs an engine; md4c is native. See [docs/NATIVE.md](docs/NATIVE.md). ## Theming @@ -145,14 +150,14 @@ import type { PartialTheme } from 'react-native-selectable-markdown'; const theme: PartialTheme = { colors: { text: '#101418', link: '#0B6E6A', codeBackground: '#F1F3F6' }, - fonts: { baseSize: 16, lineHeight: 1.5, family: 'Inter' }, + fonts: { baseSize: 16, lineHeight: 1.5, body: 'Inter' }, spacing: { blockGap: 14, listIndent: 20 }, }; ``` -Overrides merge one level deep over a base theme. The groups are `colors`, `fonts`, `spacing`, `code`, `quote`, `table`, `headings`, `rule` and `glyphs`. `colorScheme` is `'light'`, `'dark'` or `'auto'` (the default; follows the system). `mergeTheme(overrides, base)` computes a theme ahead of render. `glyphs` (bullet and task markers) are part of the projected text, so changing them shifts selection offsets; the library handles that. +Overrides merge one level deep over a base theme. The groups are `colors`, `fonts`, `spacing`, `code`, `quote`, `table`, `headings`, `rule` and `glyphs`. `colorScheme` is `'light'`, `'dark'` or `'auto'` (the default; follows the system). `mergeTheme(overrides, base)` computes a theme ahead of render. `glyphs` (bullet and task markers) are part of the projected text, so changing them shifts selection offsets; the library handles that. The font keys are `body`, `mono`, `baseSize`, `lineHeight`, `strongWeight` and the optional `strongFamily`; an unknown token is ignored, with a DEV warning. Four tokens also cross groups, each only when the token it feeds is not itself overridden: `code.borderRadius` feeds `table.borderRadius` (squaring your code blocks squares your tables), and the deprecated `spacing.quoteIndent`, `spacing.tableCellPadding` and `colors.quoteBar` feed `quote.indent`, `table.cellPaddingH`/`cellPaddingV` and `quote.barColor`. To style one mark rather than a construct (a heading ramp, a bold face instead of a weight bump), pass `attributeForMark`. Give it a stable identity; it takes part in the per-run memo. @@ -169,7 +174,9 @@ Known divergence: standalone blocks approximate list indentation with spaces ins ## Defaults for model output -No options (or `presets.commonmark`) is pure CommonMark. Use `presets.llmChat` for model output. `presets.everything` turns on every extension, spoilers included; don't feed it untrusted text. +No options (or `presets.commonmark`) turns every extension off — not a spec-conformance mode: `html` still defaults to `'strip'` and destinations are still allowlisted, both of which the table below covers. Use `presets.llmChat` for model output. `presets.everything` turns on every extension, spoilers included; don't feed it untrusted text. + +`extensions` replaces rather than extends, so an options literal naming one flag turns the rest off and a shallow spread of a preset does not help. `withOptions(preset, overrides)` layers on a preset instead, merging `extensions` and `urlPolicy` field by field; the two prefix arrays still replace deliberately. | Option | Default | `llmChat` | Notes | | --- | --- | --- | --- | @@ -178,57 +185,66 @@ No options (or `presets.commonmark`) is pure CommonMark. Use `presets.llmChat` f | `spoilers` | off | off | On only in `everything`. Parses only a balanced `\|\|x\|\|` inside one paragraph or heading; a stray `\|` stays text. | | `underline` | off | off | `_` stops meaning emphasis. On only in `everything`. | | `smartPunctuation` | off | off | Smart quotes and dashes in prose only; code and URLs stay byte-exact. On in `everything`. | -| `html` | `'strip'` | `'strip'` | `'raw'` keeps it. | -| Link schemes | `https:`, `http:`, `mailto:` | same | Anything else renders as plain text, not a dead link. Your list replaces this one; spread `DEFAULT_LINK_PREFIXES` to keep it. | -| Image schemes | `https:` | same | Blocked images render their alt text. | -| `urlPolicy.blockedLinks` | `'text'` | `'text'` | `'node'` keeps a blocked link as a `blocked: true` node for your renderer. Never navigable. | +| `html` | `'strip'` | `'strip'` | `'strip'` drops an HTML block with the lines it covers — a `
`, a `
` or a raw `` contributes nothing to the document — and drops inline tags while keeping the text between them. `
` is the exception: it becomes a hard break spanning the tag. `'raw'` keeps both as `htmlBlock` / `htmlSpan` nodes. | +| Link prefixes | `https://`, `http://`, `mailto:` | same | A case-insensitive prefix test, not scheme parsing: `https:example.com` matches nothing and renders as plain text, not a dead link. Your list replaces this one; spread `DEFAULT_LINK_PREFIXES` to keep it. A prefix reaching into a path (`myapp://checkout/`) is a scope: a destination whose path climbs back out of it with `..` (raw or percent-encoded once) is refused, one that descends and returns (`a/../b`) is not, and a `..` inside a query string or fragment is left alone. | +| Image prefixes | `https://` | same | Blocked images render their alt text. | +| `urlPolicy.blockedLinks` | `'text'` | `'text'` | `'node'` keeps a blocked link as a `blocked: true` node for your `onLinkPress`, `embed` or `classifyBlock` handler — the `link` renderer runs only for standalone blocks. Never navigable. | ### Custom link schemes The allowlist runs at parse time, so `[1](#citation-1)` or `[record](fhir://...)` collapse into plain text by default. Two ways to keep them: -- If the destination should open, add its prefix: `urlPolicy: { linkPrefixes: [...DEFAULT_LINK_PREFIXES, 'tel:'] }`. -- If it is an in-app identifier, set `blockedLinks: 'node'` and render it yourself. Nothing you don't claim can navigate. +- If the destination should open, add its prefix: `withOptions(presets.llmChat, { urlPolicy: { linkPrefixes: [...DEFAULT_LINK_PREFIXES, 'tel:'] } })`. +- If it is an in-app identifier, set `blockedLinks: 'node'` and handle it yourself. Nothing you don't claim can navigate. -```tsx -import { defaultRenderers } from 'react-native-selectable-markdown'; -import type { EngineOptions, RendererOverrides } from 'react-native-selectable-markdown'; +The `renderers` prop overrides how a node kind draws, and those overrides run for standalone blocks only. Paragraphs, headings, lists, quotes and — since they became prose kinds — code blocks, tables, rules and HTML blocks flow into a native run drawn from projected text and attributes, so a `link`, `codeBlock` or `table` override is dead there until `classifyBlock` claims the block back (DEV warns when `blockedLinks: 'node'` meets a `link` override and no other channel). What does reach a flowing run: the theme and `attributeForMark` for styling, `onLinkPress` for taps, `embed` for a real element. Links inside a run are native tappable ranges, and taps arrive at `onLinkPress({ href, blocked, start, end })`; without a handler, live links open through `Linking.openURL` and blocked ones do nothing. -// Keep the annotation, or TS widens 'node' to string. -const options: EngineOptions = { urlPolicy: { blockedLinks: 'node' } }; +Each renderer function is its own React component type, which is what makes `renderers={editing ? draft : read}` safe: swapping a renderer unmounts the old one and mounts the new one, so two renderers never share a hook list. The cost is the ordinary React one — a renderer written as an arrow literal inside JSX is a new function every render and remounts its subtree every render — so give the map and the functions in it a stable identity (module scope, or `useMemo`/`useCallback`). -const renderers: RendererOverrides = { - link: (node, ctx) => { - const m = node.blocked ? /-citation-(\d+)$/.exec(node.href) : null; - return m ? : defaultRenderers.link(node, ctx); - }, -}; +```tsx +import { openUrl, presets, withOptions } from 'react-native-selectable-markdown'; + +const options = withOptions(presets.llmChat, { urlPolicy: { blockedLinks: 'node' } }); + + { + const citation = blocked ? /^#citation-(\d+)$/.exec(href) : null; + if (citation) showCitation(Number(citation[1])); + else if (!blocked) openUrl(href); + }} +/> ``` -Inside a selection run, links are native tappable ranges and the `link` renderer does not run. Taps arrive at `onLinkPress({ href, blocked, start, end })`. Without a handler, live links open through `Linking.openURL` and blocked ones do nothing. +`openUrl` re-checks its argument against `DEFAULT_LINK_PREFIXES`, so pass your own list as its second argument (`openUrl(href, linkPrefixes)`) when `urlPolicy.linkPrefixes` is not the default — the built-in press handler does. -A renderer override changes what a node draws, not which selection run it lives in. To keep a real card *inside* the sweep, claim it through `embed`: the node projects as one placeholder character, the host reserves your declared size there and reports where it landed, and your element is overlaid on that space. Selecting across the card copies its exact markdown; `copy-text` substitutes the `text` you declare. +A renderer override changes what a node draws, not which selection run it lives in. To keep a real card *inside* the sweep, claim it through `embed`: the node projects as one placeholder character, the host reserves your declared size there and reports where it landed, and your element is overlaid on that space. Selecting across the card copies its exact markdown; `copy-text` substitutes the `text` you declare. Both this example and the `classifyBlock` one below assume the `options` above — under the default policy a `cards:` or `widget:` link has already collapsed to text before any claim or classifier sees it. ```tsx import type { EmbedRenderer } from 'react-native-selectable-markdown'; // Module scope or useCallback: a changed claim resegments and reprojects. -const embed: EmbedRenderer = (node, { topLevel }) => +const embed: EmbedRenderer = (node) => node.kind === 'link' && /^cards:/.test(node.href) ? { - width: topLevel ? 320 : 160, // declared, not measured: sizing is layout-affecting + width: 160, // declared, not measured: sizing is layout-affecting height: 88, text: '[cards]', // what copy-text shows for the card - render: (node) => , + render: () => , // closes over the narrowed node } : undefined; ``` -A block-level embed may be any height; an inline one shares a line with prose, so keep it chip-sized (on iOS a line cannot outgrow its paragraph's leading). `topLevel` is false for a node nested under a list item or blockquote, where a full-column reservation would overflow the leading margin. The card owns taps inside its bounds, so a long-press on it starts no selection. +A block-level embed may be any height; an inline one shares a line with prose, so keep it chip-sized (on iOS a line cannot outgrow its paragraph's leading). `topLevel` is true only for a block that is a direct child of the document — the one position where a full-column-width reservation is safe; the size is whatever the claim declares either way. Everything else is offered `topLevel: false`: a code block inside a list item, a table inside a blockquote, and every inline, a link in a top-level paragraph included. The card owns taps inside its bounds, so a long-press on it starts no selection. No overlay mounts while its run is the unsettled streaming tail — repair rewrites that text every tick — and the reservation is native either way, so nothing reflows when the run settles and the card appears. + +`EmbedSpec.render` carries the same identity rule as a renderer: it is the overlay's component type, so switching `render` for a span remounts the card — the point — while an arrow rebuilt on every claim remounts it on every reprojection. Give it a stable identity, reading what it needs off the `node` it is handed, when the card holds state. -`classifyBlock` marks a block `'standalone'` so it gets its own selection scope and renderer. Use it for blocks that own a competing gesture and should end the sweep rather than flow through it. Images and spoilers are standalone already. Give it a stable identity; the document is resegmented when it changes. +`images` decides how pictures ride along. The default `'embed'` claims each image, reserves `spacing.imageWidth` × `spacing.imageHeight` (280 × 200 points) and overlays `renderers.image` on it, so an illustrated paragraph keeps the sweep; `'standalone'` sends the containing block to the renderer path instead, which is what you want for full-bleed or intrinsically sized pictures, or when a streamed image must draw before its run settles. + +`classifyBlock` marks a block `'standalone'` so it gets its own selection scope and renderer. Use it for blocks that own a competing gesture and should end the sweep rather than flow through it. A block holding a spoiler, or an image no claim covered, is standalone already. Give it a stable identity; the document is resegmented when it changes. ```tsx import type { ClassifyBlock } from 'react-native-selectable-markdown'; @@ -243,10 +259,10 @@ const classifyBlock: ClassifyBlock = (node) => markdown source │ ▼ -engine (md4c, or your own) ──► ParsedDocument: a SourceSpan on every node +stream layer (streaming only): settled prefix + repaired tail ──► parse input │ ▼ -stream layer: settled-prefix tracking + tail repair +engine (md4c, or your own) ──► ParsedDocument: a SourceSpan on every node │ ▼ selection runs: adjacent flowing blocks merged, projected to display text + decorations @@ -270,18 +286,21 @@ selection offsets ──► mapSelectionToSource ──► exact source span ─ | Area | Where it stands | | --- | --- | | Parser | md4c, 651/652 on CommonMark 0.31.2 (the one failure is example 174, an unclosed HTML block inside a blockquote). The only parser; throws where not linked. | -| Streaming | Incremental tail-only parsing, checked by a prefix oracle. Coalescing, holdback, smoothers. A 147-case tail-repair corpus. | -| Selection and copy | Exact source ranges, property-tested. Code blocks, tables and rules flow through runs; images and spoilers are standalone. | -| Copy menu | Copy Text and Copy Markdown. Custom items need iOS 16+; iOS 13.4 to 15 gets the system menu only. | -| Selection host | Fabric only (`react-native >= 0.82`). CI compiles the C++ against real renderer headers and the Swift against the iOS SDK, but there is no example app yet, so on-device behaviour is reviewed rather than exercised. | -| Embeds | The `embed` prop: a claimed node flows through its run as one placeholder, the host reserves its declared size and reports the rect (`onEmbedLayout`), JS overlays the element. Removed in 0.10.0, restored in 0.11.0. Reviewed on-device like the host itself. | +| Streaming | Incremental tail-only parsing, checked by a prefix oracle. Coalescing, holdback, smoothers. A 251-case tail-repair corpus. | +| Selection and copy | Exact source ranges, property-tested. Code blocks, tables and rules flow through runs; a block holding a spoiler, or an image no claim covered, is standalone — the whole block, not just the construct. Selections never span hosts: a document's runs clear another's unless `exclusiveSelection={false}` opts them out in both directions — which keeps every range reported and exact for copy, but only the run holding focus draws a highlight. | +| Copy menu | Copy Text and Copy Markdown, retitleable from JS and extensible with your own ids. With no title from JS the labels come from platform resources (`NSLocalizedString`, `res/values/strings.xml`), which the host app can override. Custom items need iOS 16+; iOS 13.4 to 15 gets the system menu only. | +| Imperative selection | `onSelectionChange`, and a ref with `getSelection()`, `clearSelection()` and `setSelection(span)`. Codegen commands on both hosts; reviewed rather than exercised on device, like the host itself. No scroll-to-span. | +| Selection host | Fabric only (`react-native >= 0.82`). CI compiles the C++ and runs codegen against the pinned RN 0.75.4 in `devDependencies` — below the peer floor, since that bump has not landed — and the Swift against the iOS SDK. There is no example app yet, so on-device behaviour is reviewed rather than exercised. | +| Accessibility | Links, headings, list items and table cells survive run merging: each is a VoiceOver/TalkBack focus stop, a link activates through the same press path a tap takes, a heading carries the platform heading trait, and on Android an item or cell carries its position (`CollectionItemInfoCompat`), which iOS has no trait for. Code-block and blockquote structure is still flattened by merging — neither platform has a primitive for it. `accessible` or `accessibilityRole` on the container collapses the document to one element, so label it but do not make it a leaf. Standalone blocks keep the roles `renderers.tsx` sets. Reviewed, not exercised: no screen reader has run against it here. | +| Embeds | The `embed` prop: a claimed node flows through its run as one placeholder, the host reserves its declared size and reports the rect (`onEmbedLayout`), JS overlays the element. Images are claimed this way by default. Removed in 0.10.0, restored in 0.11.0 ([CHANGELOG.md](CHANGELOG.md) is the record; the 0.10.0 release notes are one squashed commit). Reviewed on-device like the host itself. | +| Package surface | The entry names every export instead of re-exporting modules wholesale, so internals (the native decoder, the selection-action codec, `classifyTopLevelBlock`) moved to deep paths under `dist/`, which the `exports` map declares and `verify:pack` resolves. `withOptions(preset, overrides)` composes options without flattening a preset. | | Android selection preservation | Not implemented, and the largest known gap. Each streamed text swap drops the selection. iOS preserves it. | | Benchmarks | Node harnesses in `bench/`. On-device numbers are planned. | | Example app | Planned. | ## Benchmarks -Measured 2026-09-01 on an Apple M2 Max with an arm64 Node 22, from the harnesses in `bench/`. Markdown-to-HTML over a 289 kB spec-derived corpus, one fresh process per library, all in the same run: this package 15.8 MB/s, commonmark.js 10.0, marked 9.4, markdown-it 6.2. The number for this package includes building the span-carrying AST and serializing it to HTML. Streaming appends parse at most 277 characters regardless of stream length and cost about 2.7 µs each end to end. These are V8 numbers; on Hermes the JS decode (43 to 53% of a parse) will be slower. Methodology and full tables in [docs/BENCHMARKS.md](docs/BENCHMARKS.md). +Measured on an Apple M2 Max with an arm64 Node 22, from the harnesses in `bench/`: the cross-parser comparison is the 2026-09-01 run, everything else a 2026-09-02 re-run. Markdown-to-HTML over a 289 kB spec-derived corpus, one fresh process per library, all in the same run: this package 15.8 MB/s, commonmark.js 10.0, marked 9.4, markdown-it 6.2. The number for this package includes building the span-carrying AST and serializing it to HTML. `engine.parse` on a 64 B tail cost 3.0 µs in the re-run — a parse alone, without the tail repair, span splice and snapshot an append also pays. What an append actually parses depends on whether the stream anchors. On the bundled sprint-review transcript it does: a blank line closes a paragraph and everything above it freezes, so appends parse a mean of 105 and at most 277 of the 1,162 final characters. A stream that never anchors gets none of that. `StreamSession.isAnchorSafe` is false for a list and for unclosed or indented code, and a blank line does not end a list, so `conformance/fixtures/transcript-giant-list.json` — 21.9 kB in 2,484 deltas, the commonest long LLM answer shape — parses a mean of 10,854 and a max of 21,881 of 21,927 characters, every append reaching the engine, and its incremental-vs-full ratio comes out above 1: tail-only parsing costs more there than reparsing the whole document per token. `npm run bench:streaming` replays both. These are V8 numbers; on Hermes the JS decode (38 to 51% of a parse) will be slower. Methodology and full tables in [docs/BENCHMARKS.md](docs/BENCHMARKS.md). ## Contributing @@ -294,7 +313,18 @@ npm run verify:pack # the packed tarball loads npm run check:codegen && npm run check:fabric-cpp && npm run check:swift ``` -Without a compiler the native suites report as skipped, not passed. CI builds the addon as a hard gate. See [native/node/README.md](native/node/README.md). +Without a compiler the native suites report as skipped, not passed. CI, the release workflow and `npm run release` all build the addon as a hard gate before the suite runs; `npm run release -- --skip-tests` opts out of the build and the suite together, and says so. See [native/node/README.md](native/node/README.md). + +## Releasing + +Publishing is tag-triggered. The tarball `npm run release` writes locally is a dry run, never the published artifact. + +1. `npm run release ` — typecheck, native addon build and `npm test` (before the bump, so a red suite leaves the tree alone), the version bump in `package.json` and `package-lock.json`, the release guard, `verify:pack`, `npm pack`. Nothing is committed, tagged or published. The guard (`scripts/check-unreleased-breaking.mjs`) runs after the bump, so it judges the bump: it fails when [CHANGELOG.md](CHANGELOG.md)'s Unreleased section names a BREAKING change and the version has not moved past the latest `v*` tag, and a failure rolls the bump back. +2. Add the version's section to [CHANGELOG.md](CHANGELOG.md) (`## [X.Y.Z] — `), moving the Unreleased entries under it. `.github/workflows/release.yml` quotes the section as the GitHub release notes and refuses to publish a version that has none. Moving the entries is required, not tidying: the preflight job runs the same release guard with the tag being pushed, so a BREAKING line left under Unreleased fails the tag in seconds — notes are read from the section for the tag, so the break would otherwise ship inside a version whose notes never mention it. +3. `git add package.json package-lock.json CHANGELOG.md && git commit -m "release X.Y.Z"`. +4. `git push origin main && git tag vX.Y.Z && git push origin vX.Y.Z`. + +The tag runs `release.yml`: the macOS gates (Swift, the iOS header set), then everything CI runs, then its own `npm pack`, the GitHub release with the tarball attached, and `npm publish --provenance --access public`. A hand `npm publish ` from a laptop is not the supported path — it ships without provenance, which only that workflow can mint, and without the release gates. `scripts/release.mjs` prints steps 2 to 4 with the version filled in. ## Prior art diff --git a/SelectableMarkdown.podspec b/SelectableMarkdown.podspec index e772e70..29ed0e3 100644 --- a/SelectableMarkdown.podspec +++ b/SelectableMarkdown.podspec @@ -28,14 +28,20 @@ Pod::Spec.new do |s| # not apply to them, and they need no C++ flags). # # THE FIRST GLOB IS `platform/ios/*` AND NOT `platform/ios/**/*`, WHICH IS - # THE POINT. platform/ios/fabric/ is added below, only under the new - # architecture. Its files are individually wrapped in - # `#ifdef RCT_NEW_ARCH_ENABLED` as well, so a recursive glob here would still - # produce empty translation units rather than errors — but "empty translation - # unit" is one preprocessor slip away from "an old-architecture app fails to - # build on a React-RCTFabric header it never asked for", and this package has - # no way to reproduce that failure. Belt and braces, as the guard comments in - # those files say. + # THE POINT — for header hygiene, not for architecture. There is no + # architecture to select any more (see the top of this file): + # platform/ios/fabric/ is added below unconditionally, in its own entry, and + # every directory named there is named again in `private_header_files`. That + # pairing is what keeps the public Objective-C surface separate from the + # headers that must never reach the umbrella header, and it is spelled one + # directory at a time. A recursive glob would sweep any subdirectory added + # later into the public set by default, and a C++ or React-RCTFabric header + # landing there is the hard build failure the next comment describes. + # + # The Fabric sources are still individually wrapped in + # `#ifdef RCT_NEW_ARCH_ENABLED`. That guard is always true for this pod — + # install_modules_dependencies below defines the macro — so it is belt and + # braces, not a gate. source_files = [ "platform/ios/*.{swift,h,m,mm}", "platform/cpp/*.{h,cpp}", @@ -97,9 +103,31 @@ Pod::Spec.new do |s| # anyway (new_architecture.rb:96 takes it from # Helpers::Constants.cxx_language_standard), and React Native's own headers # require it — react/utils/hash_combine.h:16 declares a `concept`. Naming - # it here keeps the standard explicit for `npm run check:fabric-cpp - # --syntax-only platform/cpp/*.cpp`, which is what proves the markdown - # engine still compiles clean at it. + # it here keeps the standard explicit for the check that proves the + # markdown engine still compiles clean at it: + # + # npm run check:fabric-cpp -- --syntax-only platform/cpp/*.cpp + # + # The bare `--` is load-bearing. npm forwards only what follows it, so a + # `--syntax-only` written before it is swallowed as an npm config option + # (npm 11 warns about it and carries on) and the script runs its default + # full compile instead. + # + # That command is a CI gate, not only a local one: the fabric-cpp job in + # .github/workflows/ci.yml runs it on both runners (adding + # `--platform ios` on macOS, `--platform android` on ubuntu, because an + # explicit file list narrows the script to a single pass) and + # .github/workflows/release.yml runs it on both legs of the release. The + # default `npm run check:fabric-cpp` skips platform/cpp on purpose — it + # reaches no react/renderer header — so before that step existed, the + # engine and its JSI installer compiled in no automated job at all. + # + # Both workflows build that file list with `find`, not with the glob + # written above: `platform/cpp/*.cpp` is expanded by the shell and does not + # descend, so a source added in a subdirectory of platform/cpp would fall + # outside the gate with nothing going red. The glob is fine for a local + # run, where the sources are in front of you; it is not what a gate should + # rest on. "CLANG_CXX_LANGUAGE_STANDARD" => "c++20", "DEFINES_MODULE" => "YES", # Every entry is a bare-name include somewhere in the tree, so it has to @@ -133,9 +161,8 @@ Pod::Spec.new do |s| # SelectableMarkdownModule.mm reaches the runtime through selectors that # both RCTCxxBridge and the bridgeless RCTBridgeProxy implement, so it # needs no header out of React-NativeModulesApple or React-runtimeexecutor - # — which also means the old-architecture install of this pod cannot drag - # new-arch pods into an old-architecture app, and cannot fail to build when - # they are absent. + # — which is why it compiles the same way whether the host app runs with a + # bridge or bridgeless, and why neither pod has to be named here. "HEADER_SEARCH_PATHS" => [ "$(inherited)", '"$(PODS_TARGET_SRCROOT)/platform/cpp"', diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index c1e2d0e..0ee5fe8 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -108,7 +108,7 @@ find_package(ReactAndroid CONFIG QUIET) if(NOT ReactAndroid_FOUND) message(FATAL_ERROR "react-native-selectable-markdown: the ReactAndroid prefab was not found. " - "This library needs React Native >= 0.73 with android.buildFeatures.prefab " + "This library needs React Native >= 0.82 with android.buildFeatures.prefab " "enabled in the consuming module.") endif() @@ -118,14 +118,20 @@ endif() # else, so libjsi.so satisfies every undefined symbol we produce. # # The reason this matters is version reach. `jsi` is the one prefab module -# React Native has published unchanged across every version this package -# supports — verified in ReactAndroid/build.gradle(.kts) on 0.73, 0.75, 0.76, -# 0.79 and 0.81. Everything else moved: 0.73-0.75 published a target per -# feature (react_nativemodule_core, reactnativejni, turbomodulejsijni, ...) -# and 0.76 collapsed the lot into a single `reactnative`. Linking any of them -# "in case we need it later" would buy nothing today and would make this file -# need a version ladder — so it does not, and if a CallInvoker is ever -# genuinely needed the ladder can be added then, with a symbol to justify it. +# React Native has published under an unchanged name across every version this +# package has ever supported: read out of ReactAndroid/build.gradle(.kts) on +# 0.73, 0.75, 0.76, 0.79 and 0.81 back when the peer floor was 0.73, and on +# 0.75.4 — the version this repository itself builds against — since. The +# floor package.json declares today is >= 0.82, and nobody re-reads that file +# on every release, which is precisely why the TARGET test below is a hard +# error rather than an assumption: a rename or a removal stops the build with +# a message naming the module. Everything else moved: 0.73-0.75 published a +# target per feature (react_nativemodule_core, reactnativejni, +# turbomodulejsijni, ...) and 0.76 collapsed the lot into a single +# `reactnative`. Linking any of them "in case we need it later" would buy +# nothing today and would make this file need a version ladder — so it does +# not, and if a CallInvoker is ever genuinely needed the ladder can be added +# then, with a symbol to justify it. # # FABRIC DOES NOT CHANGE THAT, AND THIS IS THE PART PEOPLE EXPECT TO BREAK. # The Fabric shadow node, its component descriptor and the JNI measurer are @@ -149,7 +155,7 @@ endif() if(NOT TARGET ReactAndroid::jsi) message(FATAL_ERROR "react-native-selectable-markdown: ReactAndroid::jsi is missing from the " - "React Native prefab. Expected React Native >= 0.73.") + "React Native prefab. Expected React Native >= 0.82.") endif() target_link_libraries(selectable-markdown PRIVATE ReactAndroid::jsi) diff --git a/android/build.gradle b/android/build.gradle index ebd89b5..390f47a 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -57,14 +57,13 @@ apply plugin: 'org.jetbrains.kotlin.android' // // WHAT APPLYING IT BUYS. ReactPlugin.kt:91-93 runs configureCodegen(isLibrary // = true) for every com.android.library that applies it, and the onlyIf guards -// on both codegen tasks (:177, :201) pass for libraries REGARDLESS OF -// ARCHITECTURE. So the generated SelectableRunHostManagerInterface is on the -// classpath in old-architecture apps too, which is what lets -// SelectableRunHostViewManager implement it unconditionally — the single -// compile-time link between src/view/SelectableRunHostNativeComponent.ts and -// the Kotlin setters. Rename a prop in the spec without renaming the setter -// and this module stops compiling, instead of shipping a prop that silently -// never arrives. +// on both codegen tasks (:177, :201) pass for every library. So the generated +// SelectableRunHostManagerInterface is on the classpath in any app that can +// build this module, which is what lets SelectableRunHostViewManager implement +// it unconditionally — the single compile-time link between +// src/view/SelectableRunHostNativeComponent.ts and the Kotlin setters. Rename +// a prop in the spec without renaming the setter and this module stops +// compiling, instead of shipping a prop that silently never arrives. apply plugin: 'com.facebook.react' def safeExtGet(prop, fallback) { @@ -197,4 +196,17 @@ repositories { dependencies { // Version is resolved by the host app's React Native gradle plugin. implementation 'com.facebook.react:react-android' + // `androidx.customview.widget.ExploreByTouchHelper`, the virtual-view + // provider RunAccessibility.kt puts over a run's links and headings. + // + // IT ALREADY ARRIVES TRANSITIVELY, and is named anyway. React Native's + // own artifact publishes appcompat as an `api` dependency, and + // appcompat -> drawerlayout -> customview, which is the only reason + // React Native's `ReactAccessibilityDelegate` can extend the same class + // without declaring it either. Relying on a two-hop transitive for a + // type this module imports directly is how a library breaks on somebody + // else's dependency upgrade, so the coordinate is stated here. The + // version is a floor, not a pin: Gradle resolves the highest across the + // graph, and 1.1.0 has been the current stable since 2020. + implementation 'androidx.customview:customview:1.1.0' } diff --git a/android/src/main/java/com/selectablemarkdown/RunAccessibility.kt b/android/src/main/java/com/selectablemarkdown/RunAccessibility.kt new file mode 100644 index 0000000..f72af51 --- /dev/null +++ b/android/src/main/java/com/selectablemarkdown/RunAccessibility.kt @@ -0,0 +1,572 @@ +package com.selectablemarkdown + +import android.graphics.Rect +import android.os.Bundle +import android.view.View +import android.widget.TextView +import androidx.core.view.accessibility.AccessibilityNodeInfoCompat +import androidx.core.view.accessibility.AccessibilityNodeProviderCompat +import androidx.customview.widget.ExploreByTouchHelper +import kotlin.math.ceil +import kotlin.math.floor + +/** + * One range of a run a screen reader has to be able to reach on its own. + * + * `pressable` non-null is the LINK case, and it carries the identifier the + * activation echoes back — so a TalkBack activation and a finger tap emit the + * same `onInlinePress` payload through the same emitter, and JS cannot tell + * which one fired. `pressable` null is a BLOCK ROLE, named by `role`: + * announced as a heading, or given a position inside its list or table, and + * never clickable. + * + * The coordinates are ONE-BASED, exactly as they cross the wire (0 is the + * absent sentinel there), and become zero-based only where + * `CollectionItemInfoCompat` is built. + */ +internal data class RunAccessibilityNode( + val start: Int, + val end: Int, + val pressable: SelectableRunHostView.Pressable?, + val role: String? = null, + val row: Int? = null, + val rowCount: Int? = null, + val column: Int? = null, + val columnCount: Int? = null, +) + +/** + * The screen-reader ranges of a run, read off the props it already has. + * + * WHY A RUN NEEDS THIS AT ALL. A run is one platform text view holding what + * the document had as several blocks, so the block-level semantics the JS + * renderer tree sets (`accessibilityRole="header"` on a heading, + * `accessibilityRole="link"` on a link — src/view/renderers.tsx) never run + * for a block that flows: TalkBack reached a heading inside a run as prose in + * a larger font, unreachable by heading-by-heading navigation, and a link as + * text that could not be activated. Both are recovered here, and both from + * fields that are genuinely on the wire. + * + * ONE NODE PER CONSTRUCT, OVER ITS OWN TEXT. A `listItem` entry stops where + * the sublist or table inside it begins, because JS narrows it there + * (`resolveRunSemantics` in src/view/runAttributes.ts). Without that this + * helper vended the parent AND its children over overlapping ranges, so + * TalkBack read a sublist twice — once inside the parent's node and again as + * its own — while iOS, which reads its ranges back off attribute runs, lost + * the children's focus stops entirely to the parent's value. The narrowing is + * what makes the two hosts announce the same thing. + * + * LIST AND TABLE STRUCTURE COMES BACK THE SAME WAY, through + * `CollectionItemInfoCompat` on the item's node and `CollectionInfoCompat` on + * the host's. That pair is what makes TalkBack say "item 2 of 5" or "row 2, + * column 3" — in the reader's own language, from numbers alone, so this + * library ships no announcement strings of its own. It is the same reason + * only three roles exist: a code block and a blockquote have no such + * primitive, so announcing them would mean shipping the English word for + * them. They stay flat, docs/SELECTION.md says so, and `RunSemanticRole` in + * src/view/runAttributes.ts is where the next role would be added. + * + * THE COST IS THAT A LIST IS READ TWICE: the TextView announces the whole + * run and then each item announces itself. That is the shape React Native's + * own `ReactAccessibilityDelegate` ships for the links inside a ``, and + * the alternative — rewriting what the TextView announces — would take the + * text-granularity navigation and selection that live on its real text with + * it. iOS, where the elements are separate objects from the text view, elides + * the covered ranges instead (`SelectableRunHostView.accessibilityElements`). + */ +internal object RunAccessibility { + + /** The `role` values this binary understands — see `RunSemanticRole` in + * src/view/runAttributes.ts. Anything else is ignored, which leaves the + * range announced as the prose it already was. Not private: the helper + * below reads them back off a resolved node. */ + const val ROLE_HEADING = "heading" + const val ROLE_LIST_ITEM = "listItem" + const val ROLE_TABLE_CELL = "tableCell" + + /** + * Link ranges come from `pressables` exactly — they are already the + * 'link' and 'blockedLink' marks, non-overlapping and sorted + * (src/view/runPressables.ts), which is the same list the tap hit-test + * uses. + * + * HEADING RANGES COME FROM THE `role` FIELD ON THE ATTRIBUTE ENTRIES, + * and that field is the whole point. They used to be INFERRED from the + * shape of the styling — size + lineHeight + weight, no family, no + * background, and not the first full-cover sized entry — because nothing + * on the wire said "heading". The inference was correct for the theme + * path and wrong in both directions for anyone using `attributeForMark`: + * a consumer whose override gave some other range that same three-field + * shape got it announced as a heading, and one that restyled headings + * without a `lineHeight` lost the announcement entirely. Neither failure + * was visible to any test in this repository. `resolveRunAttributes` now + * states the role outright, outside the overridable styling path, so a + * heading a consumer restyled is still a heading here. + * + * `roleLevel` IS PARSED AND DROPPED BECAUSE NO PLATFORM PRIMITIVE CARRIES + * A RANK. `AccessibilityNodeInfo.setHeading(true)` is a single boolean; + * there is no public API on it — and none on `CollectionItemInfo`, which + * is where a list item's depth would have to go — that holds a heading + * level or a nesting depth. The only vehicle left is the content + * description, and putting "heading level 2" there means shipping an + * English string this library cannot translate, which is worse than the + * reader's own localised "heading". iOS drops it for the same reason: + * `UIAccessibilityTraits.header` is a bit and not a rank + * (`SelectableRunHostView.accessibilityElements` says so on that side). + * It stays on the wire so that the day a primitive exists, consuming it + * is a change here and not a second wire change. + */ + fun resolve( + text: String, + attributes: RunAttributedText.Spec, + pressables: List, + ): List { + val length = text.length + if (length == 0) return emptyList() + + val nodes = ArrayList(pressables.size + 1) + for (pressable in pressables) { + val start = pressable.start.coerceIn(0, length) + val end = pressable.end.coerceIn(start, length) + if (end <= start) continue + nodes.add(RunAccessibilityNode(start, end, pressable)) + } + + for (attribute in attributes.attributes) { + val role = attribute.role + if (role != ROLE_HEADING && role != ROLE_LIST_ITEM && role != ROLE_TABLE_CELL) { + continue + } + // Clamped like every other reader of these offsets: they were + // computed against the text JS sent, which under prop skew can be + // a different length from the text in hand. + val start = attribute.start.coerceIn(0, length) + val end = attribute.end.coerceIn(start, length) + if (end <= start) continue + nodes.add( + RunAccessibilityNode( + start = start, + end = end, + pressable = null, + role = role, + row = attribute.roleRow, + rowCount = attribute.roleRowCount, + // A list is a ONE-COLUMN collection, which is how both the + // wire and TalkBack read it: the item's position is its + // row, and the column it omits is column 1 of 1. + column = if (role == ROLE_TABLE_CELL) attribute.roleColumn else 1, + columnCount = + if (role == ROLE_TABLE_CELL) attribute.roleColumnCount else 1, + ) + ) + } + + if (nodes.size > 1) { + // Reading order, with the block role ahead of a link that starts + // at the same offset: "Heading, Introduction" then "Introduction, + // button" reads the way the document does. + nodes.sortWith( + compareBy { it.start } + .thenBy { if (it.pressable == null) 0 else 1 } + ) + } + return nodes + } + + /** + * The grid to declare on the host, or null when no collection in `nodes` + * can be described. + * + * WHY ONLY ONE GRID CAN BE DECLARED. `CollectionItemInfo` is half of what + * TalkBack needs; the other half is a `CollectionInfo` on an ANCESTOR, + * and the only ancestor these virtual views have is the host TextView + * itself (`ExploreByTouchHelper` builds a flat tree by design). One node + * cannot describe two different grids. + * + * IT USED TO GIVE UP THE MOMENT THERE WERE TWO, and that lost the common + * case rather than an exotic one: a merged run is exactly where a list + * with a sublist, or a list beside a table, ends up, and a single sublist + * of a different length was enough to leave the whole run with no + * `CollectionInfo` — so the "item 2 of 5" this channel exists for was + * never phrased for the documents that need it most. It now picks the + * grid with the MOST cells (first-seen wins a tie) and the helper gives + * `CollectionItemInfo` only to the nodes of THAT grid + * (`RunAccessibilityHelper.onPopulateNodeForVirtualView`), so the biggest + * collection in the run is announced properly and nothing is ever phrased + * against a total that is not its own. The nodes left out keep their + * label and their focus stop; they lose only a position TalkBack had no + * way to say. + * + * A CANDIDATE MUST REALLY BE ONE GRID: every cell distinct and inside the + * declared bounds. Two three-item lists in one run claim the same three + * cells, so that shape is rejected and the next-largest is tried — one + * list of three announced twice would be worse than no total at all. + */ + fun collectionOf(nodes: List): Pair? { + // Insertion-ordered so a tie on size is broken by document order, and + // `sortedByDescending` is stable, so that order survives the sort. + val sizes = LinkedHashMap, Int>() + for (node in nodes) { + val shape = shapeOf(node) ?: continue + sizes[shape] = (sizes[shape] ?: 0) + 1 + } + if (sizes.isEmpty()) return null + for (candidate in sizes.entries.sortedByDescending { it.value }) { + if (describesOneGrid(nodes, candidate.key)) return candidate.key + } + return null + } + + /** + * The (rows, columns) grid a node claims to sit in, or null when it + * claims none — a heading, a link, or an item whose wire entry carried no + * position. + * + * The shape is the node's whole collection identity here: nothing on the + * wire names the list an item belongs to, so two collections with + * different totals are told apart by their totals and two with the SAME + * totals are told apart only by the duplicate-cell test below. + */ + private fun shapeOf(node: RunAccessibilityNode): Pair? { + if (node.row == null) return null + val rows = node.rowCount ?: return null + return Pair(rows, node.columnCount ?: 1) + } + + /** Whether the nodes of `shape` really are ONE grid of that shape: every + * cell inside the declared bounds, and no cell claimed twice. */ + private fun describesOneGrid( + nodes: List, + shape: Pair, + ): Boolean { + val seen = HashSet>() + for (node in nodes) { + if (shapeOf(node) != shape) continue + val row = node.row ?: continue + val column = node.column ?: 1 + if (row > shape.first || column > shape.second) return false + if (!seen.add(Pair(row, column))) return false + } + return seen.isNotEmpty() + } + + /** + * Whether `node` may carry a `CollectionItemInfo` given the grid the host + * declares. + * + * A node of a DIFFERENT grid must not: its "3" would be phrased against + * the declared collection's total and TalkBack would say "item 3 of 6" + * about an item that is third of two. With no grid declared at all + * nothing can be mis-phrased, so the info travels as it always did — it + * is inert without a collection on an ancestor, and it is what a future + * TalkBack reading item info on its own would want. + */ + fun carriesItemInfo(node: RunAccessibilityNode, collection: Pair?): Boolean { + if (collection == null) return true + return shapeOf(node) == collection + } +} + +/** + * The accessibility half of the inline-press channel: a virtual view per link + * and per block role — heading, list item, table cell — inside one run's + * TextView. + * + * WHY IT EXISTS AT ALL. The host detects link taps by observing raw + * MotionEvents (`SelectableRunHostView.dispatchTouchEvent`), deliberately, so + * that no movement method is installed on a widget whose selection behaviour + * has to stay stock. TalkBack does not inject touches: it activates a focused + * node with ACTION_CLICK through the accessibility API, so with only the + * gesture path a link inside a native run could not be activated at all — and + * nothing announced that it was there. This provides the missing channel + * WITHOUT touching the gesture path: no ClickableSpan, no + * LinkMovementMethod, and both routes end in the same `emitInlinePress`. + * + * ANDROIDX, NOT A NEW ARCHITECTURE. `ExploreByTouchHelper` lives in + * androidx.customview, which arrives with React Native's own Android artifact + * (react-android -> appcompat -> drawerlayout -> customview) and is what + * React Native's `ReactAccessibilityDelegate` extends to expose the links in + * a `` the very same way. android/build.gradle names it explicitly all + * the same, because this file imports it directly. + * + * IT IS ATTACHED TO THE CHILD TEXTVIEW, not to the host FrameLayout: the + * TextView is the node a screen reader focuses and the Layout the bounds are + * read from, so virtual-view coordinates and text coordinates are the same + * space, and the host stays a plain decorator. + */ +internal class RunAccessibilityHelper( + private val textView: TextView, + private val onLinkActivated: (SelectableRunHostView.Pressable) -> Unit, +) : ExploreByTouchHelper(textView) { + + /** Current ranges, indexed by virtual view id. */ + private var nodes: List = emptyList() + + /** + * The grid declared on the host node, derived once per `setNodes` rather + * than per node: `collectionOf` walks every node, and the per-virtual-view + * populate hook needs the same answer to decide whether that node's + * position can be phrased at all. + */ + private var collection: Pair? = null + + /** + * Replaces the exposed ranges. Called once per prop batch from + * `commitProps`, and with an empty list from `prepareToRecycle` — a + * recycled host that kept the previous run's link ranges would offer a + * screen reader taps on text that is no longer there, the accessibility + * cousin of the stale-selection failure that method exists to prevent. + * + * Value-compared like the prop setters on the host: Fabric re-delivers + * the whole prop map on every commit, and `invalidateRoot` on every one + * of them would be a stream of subtree-changed events at streaming rate. + */ + fun setNodes(value: List) { + if (value == nodes) return + nodes = value + collection = RunAccessibility.collectionOf(value) + // Bounds are read live in onPopulateNodeForVirtualView, but the SET of + // children is cached by the framework until it is invalidated. Costs + // nothing when no accessibility service is running — the send is + // gated on AccessibilityManager.isEnabled inside the helper. + invalidateRoot() + } + + /** + * A run with no links and no block roles must look EXACTLY like the stock + * TextView it is, so no provider is offered at all in that case — the + * same choice React Native's ReactAccessibilityDelegate makes + * (ReactAccessibilityDelegate.java:940-953), and the reason this delegate + * can be installed unconditionally in the host's constructor instead of + * being attached and detached as props change. + */ + override fun getAccessibilityNodeProvider(host: View): AccessibilityNodeProviderCompat? { + if (nodes.isEmpty()) return null + return super.getAccessibilityNodeProvider(host) + } + + /** + * The host TextView's own node, which is where a `CollectionInfo` has to + * go: `CollectionItemInfo` on an item means nothing to TalkBack without a + * collection on an ancestor, and `ExploreByTouchHelper`'s virtual views + * have exactly one — this view. + * + * ONE grid, the largest in the run (see `RunAccessibility.collectionOf`), + * because one node cannot describe two. A run that is a single list or a + * single table — the shape a flowed answer usually has — gets "item 2 of + * 5" and "row 2, column 3" out of it, phrased by TalkBack in the reader's + * own language; a run holding a list and a table announces the larger of + * the two, and the other one's items keep their label and their focus + * stop without a position. + */ + @Suppress("DEPRECATION") + override fun onPopulateNodeForHost(node: AccessibilityNodeInfoCompat) { + super.onPopulateNodeForHost(node) + val declared = collection ?: return + node.setCollectionInfo( + AccessibilityNodeInfoCompat.CollectionInfoCompat.obtain( + declared.first, + declared.second, + // Not hierarchical: a nested list is a collection of its own + // here and only one grid is ever declared, so this node never + // stands for a tree. + false, + ) + ) + } + + /** + * Explore-by-touch: which range is under the finger. Same offset lookup + * and same guards as the host's `pressableAt` — a point past the end of a + * short line must not "hit" the nearest character — with two extra rules: + * a link inside a block role wins, because the link is the node that can + * be activated and the role is only an announcement; and among block + * roles the SHORTEST wins. Block roles do not overlap as JS sends them + * today — an item's range stops where the construct inside it begins — + * so that second rule is a guard rather than a working tiebreak, and the + * answer it gives for an overlap is the more specific thing under the + * finger. + */ + override fun getVirtualViewAt(x: Float, y: Float): Int { + if (nodes.isEmpty()) return ExploreByTouchHelper.INVALID_ID + val offset = offsetAt(x, y) ?: return ExploreByTouchHelper.INVALID_ID + var block = ExploreByTouchHelper.INVALID_ID + var blockLength = Int.MAX_VALUE + for (index in nodes.indices) { + val node = nodes[index] + if (offset < node.start || offset >= node.end) continue + if (node.pressable != null) return index + val length = node.end - node.start + if (length < blockLength) { + block = index + blockLength = length + } + } + return block + } + + override fun getVisibleVirtualViews(virtualViewIds: MutableList) { + for (index in nodes.indices) { + virtualViewIds.add(index) + } + } + + /** + * `setBoundsInParent` is deprecated on AccessibilityNodeInfoCompat and is + * still the only bounds ExploreByTouchHelper reads: it throws if a child + * node leaves them unset, and derives the screen bounds from them itself. + */ + @Suppress("DEPRECATION") + override fun onPopulateNodeForVirtualView( + virtualViewId: Int, + node: AccessibilityNodeInfoCompat, + ) { + val range = nodes.getOrNull(virtualViewId) + val bounds = if (range == null) null else boundsFor(range) + if (range == null || bounds == null) { + // A range with no geometry — ellipsized away, or queried before + // the first layout. It still needs a node with non-empty bounds + // (the throw above), so it gets one with nothing to announce, + // which is what makes a screen reader skip it. + node.contentDescription = "" + node.setBoundsInParent(Rect(0, 0, 1, 1)) + return + } + node.contentDescription = textOf(range) + node.setBoundsInParent(bounds) + if (range.pressable != null) { + // No role description string: an untranslated one shipped by a + // library is worse than the platform's own word for a button, + // which every screen reader already says in the user's language. + node.className = "android.widget.Button" + node.isClickable = true + node.addAction(AccessibilityNodeInfoCompat.ACTION_CLICK) + return + } + if (range.role == RunAccessibility.ROLE_HEADING) { + // Localised by the reader itself, and what TalkBack's + // heading-by-heading navigation looks for. + node.isHeading = true + return + } + val row = range.row ?: return + // Only for the grid the host declares: a position phrased against + // another collection's total is a wrong announcement, where no + // position at all is merely a missing one (`carriesItemInfo`). + if (!RunAccessibility.carriesItemInfo(range, collection)) return + // Zero-based here and ONE-based everywhere else: the wire uses 0 as + // its absent sentinel, so the conversion has to happen somewhere and + // this is the only place that wants a zero-based index. + node.setCollectionItemInfo( + AccessibilityNodeInfoCompat.CollectionItemInfoCompat.obtain( + row - 1, + 1, + (range.column ?: 1) - 1, + 1, + // A GFM table's row 1 is its header row, always — that is a + // property of the wire (see `roleRow` in + // src/view/SelectableRunHostNativeComponent.ts), not a guess + // about the styling. Flagging it is what lets TalkBack name + // the column a cell is in, again in its own words. + range.role == RunAccessibility.ROLE_TABLE_CELL && row == 1, + ) + ) + } + + override fun onPerformActionForVirtualView( + virtualViewId: Int, + action: Int, + arguments: Bundle?, + ): Boolean { + if (action != AccessibilityNodeInfoCompat.ACTION_CLICK) return false + val pressable = nodes.getOrNull(virtualViewId)?.pressable ?: return false + // The same emitter the gesture path calls: one clamp, one event, one + // payload shape. + onLinkActivated(pressable) + return true + } + + private fun textOf(range: RunAccessibilityNode): CharSequence { + val text = textView.text ?: return "" + val start = range.start.coerceIn(0, text.length) + val end = range.end.coerceIn(start, text.length) + if (end <= start) return "" + return text.subSequence(start, end).toString() + } + + /** + * The range's focus rectangle in the TextView's own coordinates, or null + * when the layout cannot place it. + * + * A RANGE THAT WRAPS GETS ITS FIRST LINE, whole. Two reasons, both + * borrowed from React Native's version of this method: a screen reader + * activates a node at the centre of its bounds, and the centre of a box + * spanning two lines can sit outside the range entirely; and the + * announcement is the content description, not the box, so a generous + * first-line rectangle costs nothing. Taking the whole line rather than a + * half-open piece of it is also what keeps this direction-agnostic — in + * an RTL paragraph the range's leading edge is its right edge. + */ + private fun boundsFor(range: RunAccessibilityNode): Rect? { + val layout = textView.layout ?: return null + val text = textView.text ?: return null + val length = text.length + val start = range.start.coerceIn(0, length) + val end = range.end.coerceIn(start, length) + if (end <= start) return null + + val firstLine = layout.getLineForOffset(start) + val lastLine = layout.getLineForOffset(end - 1) + val left: Float + val right: Float + if (firstLine == lastLine) { + // An offset sitting exactly on a line break resolves to the NEXT + // line's leading edge, which would give the range a box the width + // of the paragraph; the line's visible end is the last offset + // still on it. + val endOnLine = minOf(end, layout.getLineVisibleEnd(firstLine)) + val startX = layout.getPrimaryHorizontal(start) + val endX = + if (endOnLine > start) layout.getPrimaryHorizontal(endOnLine) else startX + left = minOf(startX, endX) + right = maxOf(startX, endX) + } else { + left = layout.getLineLeft(firstLine) + right = layout.getLineRight(firstLine) + } + + val dx = textView.totalPaddingLeft - textView.scrollX + val dy = textView.totalPaddingTop - textView.scrollY + val top = layout.getLineTop(firstLine) + dy + val bottom = layout.getLineBottom(firstLine) + dy + if (bottom <= top) return null + val boxLeft = floor(left).toInt() + dx + val boxRight = ceil(right).toInt() + dx + // Never empty: ExploreByTouchHelper treats unset/empty parent bounds + // as a programming error. + return Rect(boxLeft, top, maxOf(boxRight, boxLeft + 1), bottom) + } + + /** + * The text offset under a point in the TextView's coordinates, with the + * guards `getLineForVertical` + `getOffsetForHorizontal` famously lack: + * the point must lie inside the line's vertical band and its horizontal + * extent, or a hover in the margin past a short line would "hit" the + * nearest character. The host's `pressableAt` applies the identical rule + * in its own coordinate space. + */ + private fun offsetAt(x: Float, y: Float): Int? { + val layout = textView.layout ?: return null + val localX = x - textView.totalPaddingLeft + textView.scrollX + val localY = y - textView.totalPaddingTop + textView.scrollY + if (localY < 0f || localY > layout.height.toFloat()) return null + val line = layout.getLineForVertical(localY.toInt()) + if (localY < layout.getLineTop(line).toFloat() || + localY >= layout.getLineBottom(line).toFloat() + ) { + return null + } + if (localX < layout.getLineLeft(line) || localX > layout.getLineRight(line)) return null + return layout.getOffsetForHorizontal(line, localX) + } +} diff --git a/android/src/main/java/com/selectablemarkdown/RunAttributedText.kt b/android/src/main/java/com/selectablemarkdown/RunAttributedText.kt index 96bbeb5..4764ec0 100644 --- a/android/src/main/java/com/selectablemarkdown/RunAttributedText.kt +++ b/android/src/main/java/com/selectablemarkdown/RunAttributedText.kt @@ -13,7 +13,6 @@ import android.text.style.LineHeightSpan import android.text.style.MetricAffectingSpan import android.text.style.StrikethroughSpan import android.text.style.StyleSpan -import android.text.style.TypefaceSpan import android.text.style.UnderlineSpan import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap @@ -58,6 +57,51 @@ object RunAttributedText { val strikethrough: Boolean, val color: Int?, val backgroundColor: Int?, + /** + * The semantics channel: what the range IS for a screen reader, as + * opposed to what it looks like — `"heading"`, `"listItem"`, + * `"tableCell"`, or null. `RunSemanticRole` in + * src/view/runAttributes.ts owns the set and says what bounds it. + * + * NOT A STYLING FIELD, and nothing in `build` reads it: it never + * becomes a span and never moves a glyph. `RunAccessibility.resolve` + * is the only consumer, which is why it rides this struct rather + * than a prop of its own — the ranges are already here, already + * parsed once per batch, already clamped by every reader. + * + * A value this binary does not recognise is carried through and + * ignored downstream, which leaves the range announced as the prose + * it already was — the same forward-compatibility rule as every + * other field here. + */ + val role: String?, + /** The role's depth where it has one — a heading's level (1-6), or a + * list item's nesting depth (1 at top level) — null otherwise. + * Parsed and then dropped, because no primitive on either platform + * carries a rank: `setHeading` is a boolean and `CollectionItemInfo` + * has no depth. `RunAccessibility.resolve` states the whole argument; + * it stays on the wire so consuming it is one change and not two. */ + val roleLevel: Int?, + /** + * The range's ONE-BASED position in its collection: a list item's + * place in its list, a table cell's row (row 1 is the header row). + * Null when the role has no position. + * + * One-based on the wire because 0 is the absent sentinel this whole + * struct uses; `RunAccessibility` subtracts one on the way into + * `CollectionItemInfoCompat`, which is zero-based. + */ + val roleRow: Int?, + /** The size of that collection — a list's item count, a table's row + * count including the header. The "of 5" half of TalkBack's "item 2 + * of 5", which TalkBack phrases in the reader's own language. */ + val roleRowCount: Int?, + /** The range's one-based column, for the one role laid out in two + * dimensions (`tableCell`). Null for a list item, which is read as a + * one-column collection. */ + val roleColumn: Int?, + /** The table's column count. Set with `roleColumn`. */ + val roleColumnCount: Int?, ) /** @@ -123,6 +167,16 @@ object RunAttributedText { // any colour format React Native accepts. color = optInt(entry, "color"), backgroundColor = optInt(entry, "backgroundColor"), + role = optString(entry, "role"), + // 0 is the absent sentinel the whole struct uses; which ordinals + // are meaningful is the role's business, not the parser's. A + // negative one is not an ordinal either, so the same test covers + // both. + roleLevel = optInt(entry, "roleLevel")?.takeIf { it > 0 }, + roleRow = optInt(entry, "roleRow")?.takeIf { it > 0 }, + roleRowCount = optInt(entry, "roleRowCount")?.takeIf { it > 0 }, + roleColumn = optInt(entry, "roleColumn")?.takeIf { it > 0 }, + roleColumnCount = optInt(entry, "roleColumnCount")?.takeIf { it > 0 }, ) } @@ -189,6 +243,19 @@ object RunAttributedText { ): Spannable { val out = SpannableString(text) if (text.isEmpty()) return out + // The font state an entry INHERITS. React Native's font resolver + // picks a face file from family, weight and slant together, so an + // entry that states only one of the three has to be given the other + // two or it loads the wrong file (`RunTypefaceSpan`). A `` tree + // inherits them down the tree; the wire is flat, so they are + // reconstructed here from the ranges that cover the entry: JS emits + // attributes outermost-first over properly nested ranges (marks are + // sorted by start, then by descending end — `mapSelection.finish`), so + // a stack popped against the current start is exactly "what still + // covers me". Only entries that say something about the font join it, + // which is what keeps the table-cell and embed entries — appended + // after the marks, and so out of nesting order — from disturbing it. + val fontStack = ArrayList() for (attribute in spec.attributes) { // Clamp: offsets were computed against the text JS sent, which // under prop skew can differ in length from the text in hand. @@ -197,7 +264,42 @@ object RunAttributedText { if (end <= start) continue val flags = Spannable.SPAN_EXCLUSIVE_EXCLUSIVE - attribute.fontFamily?.let { out.setSpan(TypefaceSpan(it), start, end, flags) } + var family = attribute.fontFamily + var weight = attribute.fontWeight + var italic = attribute.italic + // Whether the three together differ from what already covers this + // range, which is the only case that needs a span: an entry + // resolving to the same face as the entry enclosing it would set + // a second span with an identical answer. + var faceChanged = false + if (family != null || weight != null || italic) { + while (fontStack.isNotEmpty() && fontStack[fontStack.size - 1].end <= start) { + fontStack.removeAt(fontStack.size - 1) + } + val covering = fontStack.lastOrNull() + if (covering == null) { + faceChanged = true + } else { + if (family == null) family = covering.family + if (weight == null) weight = covering.weight + // Slant only ever turns ON: `fontStyle: 'normal'` on an + // inner mark cannot un-italicize an outer one on this + // platform either, since `StyleSpan` ORs its style in. + italic = italic || covering.italic + faceChanged = family != covering.family || + weight != covering.weight || + italic != covering.italic + } + fontStack.add(FontFrame(end, family, weight, italic)) + } + // One span for the face, carrying all three: the entry's own + // values where it has them and the covering ones where it does + // not. Absent when nothing in force names a family — there is no + // face to look up then, and `RunFontWeightSpan` below still + // weights whatever the platform gave the paint. + if (faceChanged) { + family?.let { out.setSpan(RunTypefaceSpan(it, weight, italic), start, end, flags) } + } attribute.fontSizeSp?.let { out.setSpan( AbsoluteSizeSpan(PixelUtil.toPixelFromSP(it).toInt()), @@ -214,10 +316,13 @@ object RunAttributedText { flags, ) } - attribute.fontWeight?.let { weight -> + // The entry's OWN weight, not the inherited one: an entry that + // states no weight must not re-apply its ancestor's over a range + // the ancestor's span already covers. + attribute.fontWeight?.let { declared -> if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - out.setSpan(RunFontWeightSpan(weight), start, end, flags) - } else if (weight >= 600) { + out.setSpan(RunFontWeightSpan(declared), start, end, flags) + } else if (declared >= 600) { out.setSpan(StyleSpan(Typeface.BOLD), start, end, flags) } } @@ -230,6 +335,18 @@ object RunAttributedText { } } + // The line height each embed reserves, set here — after the + // attribute spans, before the decorations — because it stands in for + // an attribute: JS sends the reservation's height as a `lineHeight` + // over the placeholder, and `RunEmbeds.applyLineHeights` replaces + // that entry with one decoded in the box's own unit (its comment has + // the whole argument). Ordering is the contract among LineHeightSpans: + // it must come after the base and mark line heights it may raise but + // must not shrink below, and before `RunDecorations`' row padding, + // which ADJUSTS what a line height assigned and would be erased by + // one set after it. + RunEmbeds.applyLineHeights(out, embeds) + // The layout-affecting half of the decoration channel (leading // margins, tab-stop columns), after the attribute spans on purpose: // column widths are measured off `out`, so a cell must already carry @@ -242,17 +359,103 @@ object RunAttributedText { RunDecorations.applyLayoutSpans(out, decorations, paint) } - // Embed reservations last, though the position is symmetry rather - // than necessity: a ReplacementSpan supplies its metrics through - // `getSize` during measurement, so its insertion order relative to - // the LineHeightSpans above is immaterial — `chooseHeight` always - // runs after the glyph metrics are in. What DOES depend on order is - // among the LineHeightSpans themselves: the embed-height `lineHeight` - // attribute JS emits after the base one is what finally sizes the - // placeholder's line (see RunEmbedSpan for the whole story). + // The embed boxes last, and here the position really is symmetry + // rather than necessity: a ReplacementSpan supplies its metrics + // through `getSize` during measurement, before any `chooseHeight` + // runs, so its insertion order relative to the spans above is + // immaterial. The half that DOES depend on order was set above. RunEmbeds.applySpans(out, embeds) return out } + + /** + * One entry's font state while `build` walks the attribute list: what it + * declared, filled in from whatever covered it. `end` is the clamped + * offset the frame stops covering at, which is what pops it. + */ + private class FontFrame( + val end: Int, + val family: String?, + val weight: Int?, + val italic: Boolean, + ) +} + +/** + * The `fontFamily` of a range, resolved the way React Native resolves one. + * + * WHY NOT `TypefaceSpan(family)`, WHICH IS WHAT THIS USED TO BE. The framework + * span resolves through `Typeface.create(name, style)`, which reads Android's + * SYSTEM font map and nothing else, so a family shipped in `assets/fonts` + * silently rendered in the default face — while the `` + * fallback used for standalone blocks, code blocks and table cells rendered + * the same theme token in the real one. `RunTypefaces` carries the whole + * argument, including why the asset table has to be installed rather than + * passed in. + * + * FAMILY, WEIGHT AND SLANT ARE ONE QUESTION, ASKED ONCE. React Native picks + * the face FILE from all three together (`Inter_bold.ttf` for weight 700), so + * a family resolved at the wrong weight loads the wrong file. This span used + * to read the weight off the paint, which for a range declaring both had not + * been raised yet — `RunFontWeightSpan` is set after it in the same + * iteration — so `{ fontFamily: 'Inter', fontWeight: '700' }` loaded + * `Inter.ttf` and synthesized the bold, while the `` fallback + * loaded `Inter_bold.ttf`. `build` therefore hands the span the weight and the + * slant of the range it covers, inheriting whichever of the three the range + * does not state from the entries that cover it — the same inheritance a + * `` tree gets for free. + * + * WHY IT RESOLVES AT PAINT TIME rather than baking a `Typeface` in at build + * time, which would be one resolution per string instead of one per line per + * draw: the spannable is CACHED (`RunLayoutCache`), and the asset table is + * armed from a Context this object never sees. A `Typeface` baked in before + * the first `install` would freeze a system-map fallback into an entry that + * outlives the reason for it; a family NAME re-resolves correctly the moment + * the assets arrive. + * + * The paint is still consulted for what the range does not state — a weight + * no covering entry declared, and an italic set by a `StyleSpan` outside this + * builder — so the span composes as it always did rather than resetting the + * paint to a bare face. + * + * The nine-value weight scale is not decided here: React Native answers with + * one of a family's four files, and `RunFontWeightSpan` — set after it, on + * API 28+ — puts the real weight back on whatever comes back. + * `MetricAffectingSpan` because a family changes advance widths, so the + * measure path and the draw path both have to run it, which is the + * one-builder agreement the rest of this file keeps. + */ +internal class RunTypefaceSpan( + private val family: String, + private val weight: Int?, + private val italic: Boolean, +) : MetricAffectingSpan() { + + override fun updateMeasureState(paint: TextPaint) = update(paint) + + override fun updateDrawState(paint: TextPaint) = update(paint) + + private fun update(paint: TextPaint) { + // `Typeface.DEFAULT` for the bare paint is not a special case: its + // weight is 400 and it is neither bold nor italic, which is what an + // unstyled paint asks for anyway. Same idiom as RunFontWeightSpan. + val current = paint.typeface ?: Typeface.DEFAULT + paint.typeface = + RunTypefaces.resolve(family, weight ?: weightOf(current), italic || current.isItalic) + } + + /** The paint's current weight, for a range no covering entry gave one. */ + private fun weightOf(current: Typeface): Int = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + current.weight + } else if (current.isBold) { + // `Typeface.getWeight` is API 28; below it the paint carries the + // two-value trait and nothing finer, which is exactly what the + // pre-28 StyleSpan branch in `build` put there. + RunTypefaces.WEIGHT_BOLD + } else { + RunTypefaces.WEIGHT_NORMAL + } } /** @@ -269,11 +472,12 @@ object RunAttributedText { * WHY IT READS THE PAINT rather than owning a family: spans compose in * insertion order and JS emits marks outermost-first, so by the time this * runs the paint already carries the family an enclosing mark set (a strong - * span inside a code span must weight the MONO face). A TypefaceSpan built - * around a fixed typeface would reset that family; mutating the paint's - * current one composes, exactly the property StyleSpan's OR-ing had. Italic - * is carried from the current face; a fake italic (skew) lives on the paint, - * not the typeface, and is untouched. + * span inside a code span must weight the MONO face) — and, since + * `RunTypefaceSpan` resolves through React Native's asset table, that family + * may be a face loaded out of `assets/fonts`. A span holding a fixed typeface + * would reset it; mutating the paint's current one composes, exactly the + * property StyleSpan's OR-ing had. Italic is carried from the current face; a + * fake italic (skew) lives on the paint, not the typeface, and is untouched. * * MetricAffectingSpan, because weight changes advance widths: both the * measure path and the draw path run it, which is what keeps the shadow @@ -330,8 +534,11 @@ internal class RunLineHeightSpan(heightPx: Float) : LineHeightSpan { // Rounded up, once, at construction. StaticLayout works in whole pixels, // and rounding per line would let a run's height drift from the sum of its - // line heights. - private val lineHeight: Int = ceil(heightPx.toDouble()).toInt() + // line heights. Readable because `RunEmbeds.applyLineHeights` needs the + // tallest line height already covering a placeholder as the floor for the + // reservation it sets there — see that function for why a reservation may + // raise a line but never shrink one. + internal val lineHeightPx: Int = ceil(heightPx.toDouble()).toInt() override fun chooseHeight( text: CharSequence?, @@ -341,32 +548,32 @@ internal class RunLineHeightSpan(heightPx: Float) : LineHeightSpan { v: Int, fm: Paint.FontMetricsInt ) { - if (fm.descent > lineHeight) { + if (fm.descent > lineHeightPx) { // Not even the descent fits. Keep as much of it as there is room // for and give up everything above the baseline. - fm.descent = min(lineHeight.toDouble(), fm.descent.toDouble()).toInt() + fm.descent = min(lineHeightPx.toDouble(), fm.descent.toDouble()).toInt() fm.bottom = fm.descent fm.ascent = 0 fm.top = fm.ascent - } else if (-fm.ascent + fm.descent > lineHeight) { + } else if (-fm.ascent + fm.descent > lineHeightPx) { // The descent fits; keep all of it and as much ascent as is left. fm.bottom = fm.descent - fm.ascent = -lineHeight + fm.descent + fm.ascent = -lineHeightPx + fm.descent fm.top = fm.ascent - } else if (-fm.ascent + fm.bottom > lineHeight) { + } else if (-fm.ascent + fm.bottom > lineHeightPx) { // Glyphs fit; the font's extra bottom leading does not, so trim it. fm.top = fm.ascent - fm.bottom = fm.ascent + lineHeight - } else if (-fm.top + fm.bottom > lineHeight) { + fm.bottom = fm.ascent + lineHeightPx + } else if (-fm.top + fm.bottom > lineHeightPx) { // Only the font's extra top leading is left to trim. - fm.top = fm.bottom - lineHeight + fm.top = fm.bottom - lineHeightPx } else { // There is room to spare: split it evenly above and below. Rounding // up on the negative side and down on the positive one makes // bottom - top come out to exactly the requested height even when // the surplus is odd, which is what keeps a run's measured height - // equal to lineCount * lineHeight. - val additional = lineHeight - (-fm.top + fm.bottom) + // equal to lineCount * lineHeightPx. + val additional = lineHeightPx - (-fm.top + fm.bottom) val top = (fm.top - ceil(additional / 2.0f)).toInt() val bottom = (fm.bottom + floor(additional / 2.0f)).toInt() fm.top = top diff --git a/android/src/main/java/com/selectablemarkdown/RunDecorations.kt b/android/src/main/java/com/selectablemarkdown/RunDecorations.kt index a5f6cb5..f168980 100644 --- a/android/src/main/java/com/selectablemarkdown/RunDecorations.kt +++ b/android/src/main/java/com/selectablemarkdown/RunDecorations.kt @@ -114,6 +114,95 @@ object RunDecorations { ) } + /** + * The vertical room a run needs BEYOND its own text, in **dp**: `top` + * above its first line, `bottom` below its last. + * + * WHY A RUN-EDGE BOX IS DIFFERENT FROM EVERY OTHER BOX. A box's + * `paddingTop`/`paddingBottom` normally costs no height at all, because + * the projection separates blocks with '\n\n' (docs/SELECTION.md) and the + * padding is painted into the blank line that leaves. A box at the EDGE + * of a run has no such line to borrow: a table that closes an answer ends + * at the run's last character, so its bottom border would land on the + * baseline of its last row, and a code block that opens one starts at + * offset 0, so its top border would be drawn through its first line. + * Neither is a corner case — "here is a table" as the closing block of a + * model's answer is the ordinary shape. + * + * ONE FUNCTION, TWO CALLERS, WHICH IS THE POINT. `RunTextMeasure.measure` + * adds `top + bottom` to the height it reports, so the view Fabric frames + * is that much taller; `SelectableRunHostView` sets the same two values + * as the child TextView's vertical padding, so the text is drawn inside + * the room that was measured for it and every `totalPaddingTop` in that + * file follows it. Deriving them twice would be the measure/draw + * disagreement `RunTextMeasure`'s header exists to prevent. + * + * LARGEST WINS, NOT THE SUM: boxes that share an edge (an island inside a + * blockquote, both starting at offset 0) are drawn from that same edge, + * so the room the deepest padding needs is the room they all need. + * + * Offsets are clamped against `textLength` exactly as `applyLayoutSpans` + * and the draw path clamp them, so a stale offset from a newer JS bundle + * asks for room at an edge it actually reaches. Returns 0/0 for the + * ordinary run, in which case the view is exactly as tall as its text. + */ + internal fun edgePaddingDp(spec: Spec, textLength: Int): EdgePadding { + if (spec.decorations.isEmpty() || textLength <= 0) return EdgePadding.NONE + var top = 0f + var bottom = 0f + for (decoration in spec.decorations) { + if (decoration.kind != "box") continue + val start = decoration.start.coerceIn(0, textLength) + val end = decoration.end.coerceIn(start, textLength) + if (end <= start) continue + if (start == 0) top = maxOf(top, decoration.paddingTop) + if (end == textLength) bottom = maxOf(bottom, decoration.paddingBottom) + } + if (top <= 0f && bottom <= 0f) return EdgePadding.NONE + return EdgePadding(top, bottom) + } + + /** The result of `edgePaddingDp`, in dp. */ + internal data class EdgePadding(val top: Float, val bottom: Float) { + companion object { + val NONE = EdgePadding(0f, 0f) + } + } + + /** + * `edgePaddingDp` in WHOLE PIXELS, which is the form both of its consumers + * have to use. + * + * WHY THE ROUNDING LIVES HERE AND NOT AT EACH CALL SITE. The two sides of + * this contract are `RunTextMeasure.measure`, which adds the room to the + * height Fabric frames the host with, and `SelectableRunHostView + * .commitProps`, which sets the same room as the child TextView's + * padding — and `View.setPadding` takes an Int. The measure side used to + * add the un-truncated float while the view truncated it, so the drawn + * band could sit up to a pixel short of the room reserved for it: two + * derivations of one number that the comments on both sides claimed were + * one. Converting once, here, is what makes that claim true. + * + * ROUNDED UP, not truncated: the padding is the room a border needs to + * clear the text, and a fraction of a pixel short is a border drawn on the + * glyphs. A whole pixel of slack at the bottom of a run is invisible. + */ + internal fun edgePaddingPx(spec: Spec, textLength: Int): EdgePaddingPx { + val dp = edgePaddingDp(spec, textLength) + if (dp.top <= 0f && dp.bottom <= 0f) return EdgePaddingPx.NONE + return EdgePaddingPx( + ceil(PixelUtil.toPixelFromDIP(dp.top)).toInt(), + ceil(PixelUtil.toPixelFromDIP(dp.bottom)).toInt(), + ) + } + + /** The result of `edgePaddingPx`, in whole pixels. */ + internal data class EdgePaddingPx(val top: Int, val bottom: Int) { + companion object { + val NONE = EdgePaddingPx(0, 0) + } + } + /** * The layout-affecting half, applied to the spannable the one builder * produced — after the attribute spans, deliberately: tab-stop columns diff --git a/android/src/main/java/com/selectablemarkdown/RunEmbeds.kt b/android/src/main/java/com/selectablemarkdown/RunEmbeds.kt index edfa5c0..1ab86b7 100644 --- a/android/src/main/java/com/selectablemarkdown/RunEmbeds.kt +++ b/android/src/main/java/com/selectablemarkdown/RunEmbeds.kt @@ -8,6 +8,7 @@ import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.ReadableType import com.facebook.react.uimanager.PixelUtil +import kotlin.math.max /** * The `embeds` prop: reserved rectangles for one run — each a @@ -64,18 +65,106 @@ object RunEmbeds { val embedId = optInt(entry, "embedId") ?: return null val width = optFloat(entry, "width") val height = optFloat(entry, "height") - // One placeholder character, a non-negative id, and a positive size: - // anything else — including the 0.0 unset sentinel the codegen - // struct documents — reserves nothing. + // One placeholder character, a non-negative id, and a size that is + // positive AND FINITE: anything else — including the 0.0 unset + // sentinel the codegen struct documents — reserves nothing. + // + // Finiteness is tested outright rather than left to `<= 0f`, which is + // false for both NaN and Infinity. NaN is JS's answer for a size + // computed from a missing measurement and Infinity is its answer for + // one divided by zero, and an infinite dp size does not degrade + // gracefully downstream: `PixelUtil.toPixelFromDIP` keeps it infinite + // and `toInt` saturates it to Int.MAX_VALUE, which would then be the + // width of a ReplacementSpan inside a measure pass. The rule this + // channel promises is that a bad entry reserves nothing. if (start < 0 || end != start + 1 || embedId < 0) return null + if (!width.isFinite() || !height.isFinite()) return null if (width <= 0f || height <= 0f) return null return Embed(start, end, embedId, width, height) } /** - * The layout-affecting half, applied to the spannable the one builder - * produced. Each valid entry replaces its placeholder's glyph with a - * fixed `width` × `height` box that draws nothing — the overlay paints. + * The line-height half of every reservation, set where the attribute it + * stands in for was set — after the attribute spans, before the + * decorations, whose row padding ADJUSTS whatever the line heights + * assigned and would be erased by a line height set after it. + * + * BOTH HALVES OF ONE RESERVATION ARE DECODED IN ONE UNIT, which is the + * whole reason this exists. A reservation is a width × height box in DIP, + * but its height also rides `attributes` as a `lineHeight` over the same + * placeholder (src/view/runAttributes.ts) — where `RunAttributedText.build` + * decodes it as SP, because SP is the right unit for every OTHER line + * height on the wire. Under a system font scale other than 1.0 the two + * halves of one number then disagreed: at scale 0.85 the line band came + * out 15% shorter than the box the overlay was told to draw, so the embed + * hung over the line below it; at 1.3 the band was 30% taller than the + * card, leaving a gap under it. A box is a box — it does not grow with the + * reader's text-size setting, and neither does the space held open for it + * — so the reservation's line height is re-decoded here through the same + * `toPixelFromDIP` the box goes through, and JS's SP-decoded twin is + * dropped. + * + * THE FLOOR IS WHY THE TWIN IS READ BEFORE IT IS DROPPED. A reservation + * may raise a line to fit but must never shrink one — an inline chip + * shorter than the prose around it would squash that prose — so what is + * set here is the taller of the box and every line height already covering + * the placeholder. That is the rule JS applies in points + * (`Math.max(embed.content.height, floor)`), restated in pixels because + * that is the only unit in which the two are comparable once the font + * scale is in play. The twin is identified by RANGE, not identity: it is + * the LAST line height covering exactly this one character, which is what + * JS emits for it (attributes first, embeds appended last). Anything + * wider — and any earlier exact-range entry, which is what a heading whose + * entire text is this one embed produces — is prose leading, and counts + * toward the floor instead. + */ + internal fun applyLineHeights(out: Spannable, spec: Spec) { + forEachReserved(out, spec) { embed -> + var twin: RunLineHeightSpan? = null + var floorPx = 0f + for (span in out.getSpans(embed.start, embed.end, RunLineHeightSpan::class.java)) { + if (out.getSpanStart(span) == embed.start && out.getSpanEnd(span) == embed.end) { + twin?.let { floorPx = max(floorPx, it.lineHeightPx.toFloat()) } + twin = span + } else { + floorPx = max(floorPx, span.lineHeightPx.toFloat()) + } + } + twin?.let { out.removeSpan(it) } + val heightPx = PixelUtil.toPixelFromDIP(embed.heightDp) + out.setSpan( + RunLineHeightSpan(max(heightPx, floorPx)), + embed.start, + embed.end, + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE, + ) + } + } + + /** + * The box half, applied to the spannable the one builder produced. Each + * valid entry replaces its placeholder's glyph with a fixed + * `width` × `height` box that draws nothing — the overlay paints. + */ + internal fun applySpans(out: Spannable, spec: Spec) { + forEachReserved(out, spec) { embed -> + out.setSpan( + RunEmbedSpan( + PixelUtil.toPixelFromDIP(embed.widthDp).toInt().coerceAtLeast(1), + PixelUtil.toPixelFromDIP(embed.heightDp).toInt().coerceAtLeast(1), + ), + embed.start, + embed.end, + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE, + ) + } + } + + /** + * The entries that really reserve something, which is what both halves + * above walk and what `SelectableRunHostView.reportEmbedRects` re-walks + * before it reports a rect: an entry that reserved nothing must report + * nothing. * * THE CHARACTER GUARD IS THE SKEW GUARD: a span is only set where the * text really carries U+FFFC. Offsets were computed against the text JS @@ -84,21 +173,13 @@ object RunEmbeds { * box would corrupt what the reader sees, where skipping the entry only * costs the reservation. */ - internal fun applySpans(out: Spannable, spec: Spec) { + private inline fun forEachReserved(out: Spannable, spec: Spec, body: (Embed) -> Unit) { if (spec.embeds.isEmpty()) return val length = out.length for (embed in spec.embeds) { if (embed.start >= length || embed.end > length) continue if (out[embed.start] != PLACEHOLDER) continue - out.setSpan( - RunEmbedSpan( - PixelUtil.toPixelFromDIP(embed.widthDp).toInt().coerceAtLeast(1), - PixelUtil.toPixelFromDIP(embed.heightDp).toInt().coerceAtLeast(1), - ), - embed.start, - embed.end, - Spannable.SPAN_EXCLUSIVE_EXCLUSIVE, - ) + body(embed) } } @@ -134,14 +215,17 @@ object RunEmbeds { * * HOW THE HEIGHT ACTUALLY LANDS. `getSize` asks for the height as ascent * (the box sits on the baseline, rising `heightPx` above it), but the final - * line extents belong to the `LineHeightSpan`s: JS sends a `lineHeight` - * attribute equal to the embed height over this same character, emitted - * AFTER the base attribute, and `RunAttributedText.build`'s insertion-order - * rule makes that `RunLineHeightSpan` the last word on the placeholder's - * line. Its surplus branch redistributes extra room evenly above and below - * the baseline, so the baseline may sit mid-line — which is why - * `reportEmbedRects` anchors the reported rect on `getLineTop`, never on - * baseline arithmetic against this span's ascent. + * line extents belong to the `LineHeightSpan`s: `RunEmbeds.applyLineHeights` + * sets a `RunLineHeightSpan` of the same DIP-decoded height over this same + * character, after every attribute line height, and insertion order is what + * lets it raise the placeholder's line to fit the box (that function says why + * the reservation decodes its own height rather than taking JS's SP-decoded + * one, and what stops it shrinking a line). Where the line ends up taller + * still — an inline chip inside taller prose leading — the surplus branch + * redistributes the extra room evenly above and below the baseline, so the + * baseline may sit mid-line, which is why `reportEmbedRects` anchors the + * reported rect on `getLineTop` and never on baseline arithmetic against this + * span's ascent. * * IMMUTABLE, AND THAT IS LOAD-BEARING: the spannable carrying this span is * shared across the measure thread and the widget (`RunLayoutCache.styledText` diff --git a/android/src/main/java/com/selectablemarkdown/RunLayoutCache.kt b/android/src/main/java/com/selectablemarkdown/RunLayoutCache.kt index 5b50651..57e7113 100644 --- a/android/src/main/java/com/selectablemarkdown/RunLayoutCache.kt +++ b/android/src/main/java/com/selectablemarkdown/RunLayoutCache.kt @@ -16,12 +16,11 @@ import java.util.concurrent.atomic.AtomicBoolean * returns byte-for-byte what a rebuild would have; the agreement doctrine in * `RunTextMeasure` is untouched, this just stops paying for it repeatedly. * - * WHY IT EXISTS. Under Fabric the C++ shadow node re-measures on every commit - * and `SelectableRunHostView.commitProps` builds the same styled string again - * on the UI thread; on paper the shadow node dirties on every prop batch. A - * streamed message recommits its settled runs many times per second with - * nothing about them changed, so the same Spannable and the same StaticLayout - * were being rebuilt from identical inputs on two threads. React Native's own + * WHY IT EXISTS. The C++ shadow node re-measures on every commit and + * `SelectableRunHostView.commitProps` builds the same styled string again on + * the UI thread. A streamed message recommits its settled runs many times per + * second with nothing about them changed, so the same Spannable and the same + * StaticLayout were being rebuilt from identical inputs on two threads. React Native's own * text stack solves this identically (TextLayoutManager's spannable cache plus * a TextMeasureCache keyed on the full layout constraints, capped at 1024). * @@ -33,10 +32,11 @@ import java.util.concurrent.atomic.AtomicBoolean * be added to the `Spec` wrappers (the key holds the lists, not the * wrappers). * - the window display metrics (`density`, `scaledDensity`), because build - * bakes them into the spans: every `AbsoluteSizeSpan` and + * bakes them into the spans: every `AbsoluteSizeSpan` and every attribute * `RunLineHeightSpan` goes through `PixelUtil.toPixelFromSP` (reads - * `scaledDensity`) and every margin/tab-stop through `toPixelFromDIP` - * (reads `density`), both off `DisplayMetricsHolder.getWindowDisplayMetrics()`. + * `scaledDensity`), and every margin, tab stop and embed reservation — + * box and line height alike — through `toPixelFromDIP` (reads `density`), + * both off `DisplayMetricsHolder.getWindowDisplayMetrics()`. * A FONT-SCALE CHANGE MUST MISS THE CACHE: without the metrics in the key, * every run after an accessibility font-size change would keep rendering * and measuring at the previous scale, silently, until eviction happened @@ -50,8 +50,9 @@ import java.util.concurrent.atomic.AtomicBoolean * onTrimMemory, is memory reclamation, not invalidation: every entry is pure * derived data, so dropping all of them costs rebuilds and nothing else.) * - * THREADING. `RunTextMeasure.measure` runs on paper's shadow thread and on - * Fabric's layout thread (through JNI); `commitProps` runs on the UI thread. + * THREADING. `RunTextMeasure.measure` runs on whatever thread Fabric calls + * the JNI measure from — the background layout thread, or the UI thread for a + * synchronous commit; `commitProps` runs on the UI thread. * Every map access is synchronized on the map — including gets, because with * `accessOrder = true` a get reorders the map. The lock is also what safely * publishes a built Spannable across threads; after `build` returns, nothing diff --git a/android/src/main/java/com/selectablemarkdown/RunTextMeasure.kt b/android/src/main/java/com/selectablemarkdown/RunTextMeasure.kt index f5c52c1..1ba0c40 100644 --- a/android/src/main/java/com/selectablemarkdown/RunTextMeasure.kt +++ b/android/src/main/java/com/selectablemarkdown/RunTextMeasure.kt @@ -4,7 +4,9 @@ import android.os.Build import android.text.Layout import android.text.Spannable import android.text.StaticLayout +import android.text.TextDirectionHeuristics import android.text.TextPaint +import android.view.View import android.widget.TextView import com.facebook.react.uimanager.PixelUtil import com.facebook.yoga.YogaMeasureMode @@ -16,37 +18,29 @@ import kotlin.math.ceil * The one place a run's text layout is configured, and the one place it is * measured. * - * WHY THIS OBJECT EXISTS. Three separate things have to agree about how a run - * lays out: the `TextView` that draws it, the paper shadow node that measures - * it for Yoga on the shadow thread, and — under Fabric — - * `SelectableRunHostViewManager.measure`, which the C++ shadow node calls - * across JNI from the layout thread. If they disagree the symptom is not a - * build error, it is text clipped at the bottom of a run: it looks like a - * rendering bug, it gets worse with every extra line, and no test in this - * repository can see it (docs/FABRIC-PLAN.md §4.3 specifies the instrumented - * test that could, and says plainly that it cannot run here). So agreement is - * not maintained by keeping three call sites in sync — there is one paint - * configuration and one `StaticLayout` construction, and all three sides go - * through this file. `RunAttributedText.build` is the matching guarantee for - * the styled string itself. `RunLayoutCache` memoizes both and does not bend - * any of this: it sits in FRONT of the one builder and this one measure, - * never beside them, so a cache hit returns what the miss path would have - * built from the same inputs. + * WHY THIS OBJECT EXISTS. Two separate things have to agree about how a run + * lays out: the `TextView` that draws it on the UI thread, and + * `SelectableRunHostViewManager.measure`, which the C++ shadow node + * (platform/fabric/RNSMRunHostShadowNode.cpp) calls across JNI from the layout + * thread. If they disagree the symptom is not a build error, it is text + * clipped at the bottom of a run: it looks like a rendering bug, it gets worse + * with every extra line, and no test in this repository can see it + * (docs/FABRIC-PLAN.md §4.3 specifies the instrumented test that could, and + * says plainly that it cannot run here). So agreement is not maintained by + * keeping two call sites in sync — there is one paint configuration and one + * `StaticLayout` construction, and both sides go through this file. + * `RunAttributedText.build` is the matching guarantee for the styled string + * itself. `RunLayoutCache` memoizes both and does not bend any of this: it + * sits in FRONT of the one builder and this one measure, never beside them, + * so a cache hit returns what the miss path would have built from the same + * inputs. * - * A FREE CONSEQUENCE WORTH STATING: because the paper shadow node and the - * Fabric measure override call the same function with the same inputs, a run - * does not change height when an app flips `newArchEnabled`. Whatever the two - * architectures disagree about, it is not this. + * UNITS. Everything here is in **pixels**, and the caller converts — which is + * why converting is not this object's job: * - * UNITS. Everything here is in **pixels**, on both architectures, and the two - * architectures differ in what they want back — which is why converting is the - * caller's job and not this object's: - * - * - Paper's Yoga tree is itself in pixels (`LayoutShadowNode` pushes - * `PixelUtil.toPixelFromDIP` values into Yoga, and React Native's own - * `ReactTextShadowNode.measure` hands `StaticLayout` the incoming width - * unconverted and returns raw layout pixels). So the paper shadow node - * returns what `measure` returns, untouched. + * - The drawing side wants pixels, because that is what a `TextView`, its + * `TextPaint` and a `StaticLayout` all speak. It takes what `measure` and + * `configurePaint` produce, untouched. * - Fabric's Yoga tree is in points. `FabricUIManager.measure` converts the * constraints to pixels on the way in — `getYogaSize` is * `PixelUtil.toPixelFromDIP(maxSize)` @@ -121,11 +115,34 @@ internal object RunTextMeasure { private const val BREAK_STRATEGY = Layout.BREAK_STRATEGY_HIGH_QUALITY private const val HYPHENATION_FREQUENCY = Layout.HYPHENATION_FREQUENCY_NONE + /** + * Paragraph direction, pinned on both sides for the same reason the break + * strategy is: the two ends resolve it from different places by default. + * A `StaticLayout.Builder` left alone uses FIRSTSTRONG_LTR; a `TextView` + * left alone inherits TEXT_DIRECTION_FIRST_STRONG, which resolves to + * FIRSTSTRONG_**RTL** when the view's layout direction is RTL — so in an + * RTL app a paragraph with no strong directional character in it (a line + * of digits, a code fence of punctuation) was measured as LTR and drawn + * as RTL, and ALIGN_NORMAL resolves against exactly that bit. + * + * The pair below is one decision written twice, because the two APIs take + * different types: `TextDirectionHeuristics.FIRSTSTRONG_LTR` is what + * `TextView.getTextDirectionHeuristic` returns for + * `View.TEXT_DIRECTION_FIRST_STRONG_LTR`. It is also what the deprecated + * pre-M `StaticLayout` constructor uses internally, so that branch needs + * nothing added to agree, and it is the heuristic React Native resolves + * its own text layouts through (TextLayoutManager.java's `isScriptRTL`). + */ + private val TEXT_DIRECTION = TextDirectionHeuristics.FIRSTSTRONG_LTR + private const val VIEW_TEXT_DIRECTION = View.TEXT_DIRECTION_FIRST_STRONG_LTR + /** * Scratch paint for the measure path, one per thread rather than one per - * call. Per thread because `measure` runs concurrently on paper's shadow - * thread and Fabric's layout thread and TextPaint is not thread-safe; - * never handed across threads or out of this function. `configurePaint` + * call. Per thread because `measure` runs on whatever thread Fabric calls + * it from — the layout thread for a background commit, the UI thread for a + * synchronous one — while the view side configures its own paint on the UI + * thread, and TextPaint is not thread-safe; never handed across threads or + * out of this function. `configurePaint` * re-runs on every measure because textSize derives from the window * metrics, which move under a font-scale or density change; nothing else * on the paint is ever written — span measurement inside StaticLayout and @@ -180,22 +197,34 @@ internal object RunTextMeasure { * writing the paint directly is equivalent to `setTextSize` here and not a * shortcut around it. * - * The three knobs below are the ones `measure` also sets on its - * `StaticLayout`; they are set here and nowhere else so that "what the - * view is configured with" and "what was measured" cannot drift apart in - * a diff. `includeFontPadding` is `TextView`'s default and is stated - * anyway, because the measure side has to name it explicitly and a default - * that is only true on one side is not agreement. + * Every knob below is one `measure` also sets on its `StaticLayout`; they + * are set here and nowhere else so that "what the view is configured with" + * and "what was measured" cannot drift apart in a diff. `includeFontPadding` + * is `TextView`'s default and is stated anyway, because the measure side has + * to name it explicitly and a default that is only true on one side is not + * agreement. * * The line-spacing pair is set to the identity for a second reason beyond * agreement: leading is owned entirely by `RunLineHeightSpan`, driven by * the `lineHeight` attribute on the wire, and a non-zero `lineSpacingExtra` * here would add to it invisibly on the drawn side only. + * + * Fallback line spacing is pinned rather than inherited, and it is the one + * knob here whose default depends on the HOST APP: `TextView` turns it on + * for itself only when the app's targetSdk is 28+, while + * `StaticLayout.Builder` defaults it off at every level. Left alone, the + * pair disagrees about any line that fell back to another font for a + * script the chosen face lacks — emoji, CJK — where the fallback face's + * taller metrics grow the drawn line but not the measured one, which is + * the clipped-text failure this object exists to prevent. React Native + * turns it on for the same reason (TextLayoutManager.java:416-418, under + * the same API-28 guard). */ fun configureTextView(view: TextView) { configurePaint(view.paint) view.includeFontPadding = true view.setLineSpacing(0f, 1f) + view.textDirection = VIEW_TEXT_DIRECTION if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { // Pre-M Android has one line-breaking algorithm and no way to // select another, so there is nothing to synchronise there and the @@ -203,6 +232,11 @@ internal object RunTextMeasure { view.breakStrategy = BREAK_STRATEGY view.hyphenationFrequency = HYPHENATION_FREQUENCY } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + // Below API 28 neither side has the knob, so both use the font's + // own metrics and agree by construction. + view.isFallbackLineSpacing = true + } } /** @@ -249,14 +283,16 @@ internal object RunTextMeasure { } // The cache consult. The key captures every input the styled string - // and the paint read — text, both specs, and the window display - // metrics — so a hit is exactly what the code below would have - // produced; RunLayoutCache's header carries the argument, including - // why the metrics must be in the key. Streaming recommits — Fabric - // re-measures every run on every commit, paper's shadow node dirties - // on every prop batch — arrive here with identical inputs and - // identical constraints, and the second lookup turns that whole case - // into a map get. + // and the paint read — text, all three specs (attributes, decorations + // and the embed reservations, which bake their own sizes into spans), + // the window display metrics and the default locale — so a hit is + // exactly what the code below would have produced; RunLayoutCache's + // header carries the argument, including why the metrics must be in + // the key and why the entry caps are not the memory bound. Streaming + // recommits — Fabric re-measures every run whose props changed on + // every commit — arrive here with identical inputs and identical + // constraints for every run the delta did not touch, and the second + // lookup turns that whole case into a map get. val key = RunLayoutCache.key(text, spec, decorations, embeds) RunLayoutCache.measurement(key, width, widthMode, height, heightMode)?.let { return it } @@ -281,27 +317,51 @@ internal object RunTextMeasure { val wrapWidth = layoutWidth.toInt().coerceAtLeast(1) val layout = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - StaticLayout.Builder + val builder = StaticLayout.Builder .obtain(styled, 0, styled.length, paint, wrapWidth) .setAlignment(Layout.Alignment.ALIGN_NORMAL) .setLineSpacing(0f, 1f) .setIncludePad(true) .setBreakStrategy(BREAK_STRATEGY) .setHyphenationFrequency(HYPHENATION_FREQUENCY) - .build() + .setTextDirection(TEXT_DIRECTION) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + // The view side pins the same bit in `configureTextView`, + // where the comment says what the two would otherwise + // disagree about. + builder.setUseLineSpacingFromFallbacks(true) + } + builder.build() } else { - // Pre-M: no builder and no break strategy to choose, on either - // side. The remaining parameters are positional — width, alignment, - // spacing multiplier, spacing add, includePad — and match the - // builder call above knob for knob. + // Pre-M: no builder, and no break strategy or fallback line + // spacing to choose on either side. The remaining parameters are + // positional — width, alignment, spacing multiplier, spacing add, + // includePad — and match the builder call above knob for knob; + // the text direction matches too, since this constructor uses + // FIRSTSTRONG_LTR internally (see TEXT_DIRECTION). @Suppress("DEPRECATION") StaticLayout(styled, paint, wrapWidth, Layout.Alignment.ALIGN_NORMAL, 1f, 0f, true) } + // The room a box at the very EDGE of the run needs, which the text + // layout above neither asks for nor knows about: a table that closes + // an answer has nothing under its bottom border, a code block that + // opens one has nothing above its top border. Everywhere else the + // padding is painted into the blank line the '\n\n' block separator + // leaves and costs nothing. `RunDecorations.edgePaddingPx` is the one + // derivation — `SelectableRunHostView` reads the same function for + // the child TextView's padding, which is what puts the text inside + // the room measured here rather than at the top of it. In PIXELS on + // both sides, and rounded once inside that function: `setPadding` + // takes whole pixels, so a float added here would reserve a fraction + // of a pixel the view could not spend. + val edge = RunDecorations.edgePaddingPx(decorations, text.length) + val contentHeight = layout.height.toFloat() + edge.top.toFloat() + edge.bottom.toFloat() + val measuredHeight = when (heightMode) { YogaMeasureMode.EXACTLY -> height - YogaMeasureMode.AT_MOST -> minOf(layout.height.toFloat(), height) - else -> layout.height.toFloat() + YogaMeasureMode.AT_MOST -> minOf(contentHeight, height) + else -> contentHeight } val output = YogaMeasureOutput.make(layoutWidth, measuredHeight) RunLayoutCache.putMeasurement(key, width, widthMode, height, heightMode, output) diff --git a/android/src/main/java/com/selectablemarkdown/RunTypefaces.kt b/android/src/main/java/com/selectablemarkdown/RunTypefaces.kt new file mode 100644 index 0000000..89731dc --- /dev/null +++ b/android/src/main/java/com/selectablemarkdown/RunTypefaces.kt @@ -0,0 +1,149 @@ +package com.selectablemarkdown + +import android.content.Context +import android.content.res.AssetManager +import android.graphics.Typeface +import com.facebook.react.common.assets.ReactFontManager + +/** + * Family name to `Typeface`, through React Native's font machinery instead of + * Android's system font map. + * + * WHY THIS EXISTS AT ALL. `fontFamily` on the wire is a React Native family + * name — the same string a consumer puts on a `` — and React Native + * resolves those through `ReactFontManager`, which looks in + * `assets/fonts/[_bold|_italic|_bold_italic].ttf|.otf` and in the + * `res/font` families an app registered with `addCustomFont`, falling back to + * the system map only when neither has the name. The framework's + * `TypefaceSpan(String)` resolves through `Typeface.create(name, style)`, + * which knows the SYSTEM map and nothing else — so a bundled family rendered + * in the default system face in the native run while the `` + * fallback (standalone blocks, code blocks, table cells) rendered it in the + * real one. That is a visible split inside one message, and because the two + * faces have different advance widths, a wrapping split too. The default + * theme hid it: its Android families are 'sans-serif' and 'monospace', both + * system names that resolve identically either way. + * + * THE ASSET MANAGER IS INSTALLED, NOT INJECTED, because the styled string is + * built by a pure object on the layout thread with no Context in reach + * (`RunAttributedText.build`). `SelectableRunHostViewManager` arms this from + * both of the points where this component first holds one — `measure` (the + * earliest, since Fabric measures before it mounts) and `createViewInstance` + * — so no measurement can be taken before resolution works. The application + * context is what is kept: an AssetManager outlives every ReactInstance, and + * so does this singleton. + * + * RESOLUTION IS AT PAINT TIME, NOT BUILD TIME (see `RunTypefaceSpan`): the + * spannable is cached by `RunLayoutCache`, and a baked `Typeface` would + * outlive the reason it was chosen, while a family NAME re-resolves. + * + * NOTHING IS CACHED HERE, AND THAT IS THE FIX RATHER THAN AN OVERSIGHT. + * `ReactFontManager` keeps its own cache — a `FontFamily` per family holding + * one `Typeface` per style — and consults the `res/font` families an app + * registered with `addCustomFont` BEFORE it looks at assets. A cache of our + * own in front of it is therefore a second answer that can only ever go + * stale: an app that registers a family lazily (a downloaded face, a theme + * chosen after `Application.onCreate`) would have had the pre-registration + * answer frozen in for the life of the process, and the `` + * fallback — which asks React Native every time — would render the same + * `fontFamily` in a different face. That is exactly the split this file + * exists to close, reintroduced one layer up. + * + * WHAT IT COSTS, stated honestly: a monitor acquisition plus two `HashMap` + * lookups per call (the custom-font map, then the family cache), where the + * cache made it a lock-free `ConcurrentHashMap` read with the monitor taken + * only on a miss. That is a real move onto the hot path — this runs once per + * face span per line per measure and per draw — and it is the price of the + * `` fallback and this path never disagreeing about a lazily + * registered family. React Native's own text stack calls `getTypeface` this + * same way on every text update, and the alternative — verifying a cached + * face against a fresh resolve — is the resolve. + * + * THE WEIGHT AND THE STYLE ARE ASKED FOR WITH THE FAMILY, IN ONE CALL, and + * that is the second half of the same fix. React Native picks the face FILE + * from the family and the weight together — `getTypeface(family, weight, + * italic, assets)` builds a `TypefaceStyle`, whose `getNearestStyle()` maps + * the CSS weight onto the `_bold` / `_italic` / `_bold_italic` file suffix — + * so a family resolved at the wrong weight loads the wrong file. This object + * used to be handed the weight the paint happened to carry, which for + * `{ fontFamily: 'Inter', fontWeight: '700' }` was still 400 when the family + * was resolved: `assets/fonts/Inter.ttf` came back and the bold was then + * SYNTHESIZED on top of it, while the `` fallback loaded + * `assets/fonts/Inter_bold.ttf`. Different face, different advance widths, + * different wrapping — the same split one level narrower. `RunTypefaceSpan` + * now carries the weight and the italic of the range it covers, inheriting + * whichever of the three the range does not state from the entries that cover + * it, so both paths ask React Native the identical question. + * + * THE NINE-VALUE SCALE SURVIVES ON TOP OF THAT. An asset family only has four + * files, so React Native answers a weight of 500 with the regular one and + * stops there; `RunFontWeightSpan` (API 28+) then applies the real weight to + * whatever face came back, which is a refinement the `` path does not + * make for asset families and cannot change which FILE was loaded. The two + * paths therefore agree about the face and this one is finer about the + * weight, where before they disagreed about the face itself. + * + * THE LOCK IS AROUND REACT NATIVE'S CACHE. `ReactFontManager` keeps plain + * `HashMap`s and is written for the UI thread; this object's callers are the + * UI thread AND the layout thread, so its calls are serialized here. That + * cannot stop React Native's own text stack from calling it concurrently from + * a third thread — it is a narrowing, not a guarantee. The critical section is + * two map lookups once a family has been resolved once, which is why holding + * it on every call rather than only on a miss is affordable; `build` keeps + * the number of calls down by setting a face span only where family, weight + * or slant actually CHANGE from the range enclosing them. + */ +internal object RunTypefaces { + + /** CSS weights, in the numeric scale `RunAttributedText.parseFontWeight` + * normalizes to. `WEIGHT_BOLD` is also the cut React Native's + * `TypefaceStyle` uses when it picks a face file. */ + const val WEIGHT_NORMAL = 400 + const val WEIGHT_BOLD = 700 + + @Volatile + private var assets: AssetManager? = null + + /** + * Arm family resolution. Idempotent and cheap enough to call on every + * measure: after the first call it is one volatile read. + */ + fun install(context: Context) { + if (assets != null) return + assets = context.applicationContext.assets + } + + /** + * The face for one family at one weight and slant, asked of React Native + * every time — see the note above on why nothing is memoized here, and + * why the three are one question and not two. Falls back to the system map + * while no Context has reached this process yet. Never null: React + * Native's resolver itself ends in `Typeface.create(family, style)`, which + * answers the default face for a name it does not know. + * + * The CSS weight goes across whole rather than pre-collapsed: React + * Native's `TypefaceStyle` needs it that way to tell an asset family + * (nearest of four files, weight not re-applied) from a `res/font` family + * registered with `addCustomFont` (one file, weight applied to it) — a + * distinction a `Typeface` style bit cannot carry. + */ + fun resolve(family: String, weight: Int, italic: Boolean): Typeface { + val assetManager = assets ?: return Typeface.create(family, nearestStyle(weight, italic)) + val fromAssets: Typeface? = synchronized(this) { + ReactFontManager.getInstance().getTypeface(family, weight, italic, assetManager) + } + return fromAssets ?: Typeface.create(family, nearestStyle(weight, italic)) + } + + /** + * The two-value `Typeface` style for a CSS weight — what the system map + * takes, for the fallbacks above. The cut is `WEIGHT_BOLD`, matching + * `ReactFontManager.TypefaceStyle.getNearestStyle`, so a fallback and a + * resolution disagree about the weight by no more than the system map + * itself does. + */ + private fun nearestStyle(weight: Int, italic: Boolean): Int { + val bold = if (weight >= WEIGHT_BOLD) Typeface.BOLD else Typeface.NORMAL + return if (italic) bold or Typeface.ITALIC else bold + } +} diff --git a/android/src/main/java/com/selectablemarkdown/SelectableMarkdownModule.kt b/android/src/main/java/com/selectablemarkdown/SelectableMarkdownModule.kt index 764e85b..c46cb07 100644 --- a/android/src/main/java/com/selectablemarkdown/SelectableMarkdownModule.kt +++ b/android/src/main/java/com/selectablemarkdown/SelectableMarkdownModule.kt @@ -27,19 +27,41 @@ import com.facebook.react.module.annotations.ReactModule * that read finds nothing and the parse fails on an app whose native * module was, a tick later, perfectly fine. * - * Why it returns Boolean and never throws: this runs on the JS thread during - * startup, and the causes it can hit (a Gradle/NDK misconfiguration, an ABI - * the host app filtered out, a JS runtime that is already gone) are things + * Why it reports an outcome and never throws: this runs on the JS thread + * during startup, and the causes it can hit (a Gradle/NDK misconfiguration, an + * ABI the host app filtered out, a JS runtime that is already gone) are things * only a rebuild can fix. Throwing here would take the whole app down at * launch for a fault no user action can clear, and would report it far from * its cause. So the reason goes to logcat for whoever is debugging the build - * and the outcome comes back as `false`. It is not a soft failure: md4c is - * the only parser this package ships, so a `false` that is never followed by + * and the outcome comes back as a string. It is not a soft failure: md4c is + * the only parser this package ships, so a refusal that is never followed by * a successful install means every non-empty document throws out of * parseDocument — with a message naming the missing native module — instead * of rendering. The view layer itself is unaffected, which is why this is * reported rather than fatal: the app runs, its markdown does not parse. * + * WHY THE OUTCOME IS THREE STRINGS AND NOT A BOOLEAN, which is what it used + * to be. A bare `false` conflated "not yet" with "not ever". The JS side is + * invited to poll `isNativeEngineAvailable()`, so it retried every failure on + * every call — each retry crossing the bridge and writing another Log.w. Only + * this class can tell the two apart, so this is where the difference is + * spoken: + * + * - `installed` — the global is on the runtime now. + * - `unavailable` — TRANSIENT. No JSI context or a null runtime pointer: the + * context is starting or tearing down. Ask again. + * - `refused` — PERMANENT for this process or this runtime: the .so did + * not load, its JNI symbol did not match, or the C++ + * installer rejected the runtime. Memoized below so the + * warning prints once, and memoized again in + * src/engine/native/install.ts so the next call does not + * even reach this method. + * + * A JS bundle older than this binary ignores the return value and reads the + * global instead, so the change is invisible to it; a newer bundle meeting an + * older binary sees `true`/`false` and reads a `false` as `unavailable`, which + * is the older behaviour — retried forever, but never wrong. + * * The @ReactModule annotation is what the new architecture reads the JS-facing * name from. ReactPackageTurboModuleManagerDelegate builds each package's * ReactModuleInfo map at startup and prefers `reactModule.name()` over @@ -53,10 +75,39 @@ class SelectableMarkdownModule(reactContext: ReactApplicationContext) : override fun getName(): String = MODULE_NAME + /** + * Remembers a PERMANENT refusal, so a polling caller gets one logcat line + * instead of one per call. + * + * Only the permanent half is memoized. `unavailable` is deliberately not, + * because that branch means "ask again once the context is up", and a + * cache there would turn a startup race into a permanent failure. + * + * Scoped to this module instance, which is the right scope: a reload + * builds a new ReactApplicationContext and therefore a new module, so a + * fresh runtime is asked afresh rather than inheriting a refusal that + * belonged to the runtime before it. `nativeLibraryLoaded` is the one + * genuinely process-wide fact, and it has its own memo below. + * + * Success is deliberately NOT memoized here, unlike the iOS module's + * `_installed`: `installSelectableMarkdown` is idempotent per runtime + * (OnLoad.cpp invariant 4) and re-installing is cheap, while a stale + * "already installed" flag surviving a runtime swap would be silent and + * wrong. src/engine/native/install.ts memoizes the success, so the second + * call does not reach this method anyway. + */ + private var refused = false + @ReactMethod(isBlockingSynchronousMethod = true) - fun install(): Boolean { + fun install(): String { + if (refused) { + return OUTCOME_REFUSED + } + if (!nativeLibraryLoaded) { - return false + // The `by lazy` below has already logged, once for the process. + refused = true + return OUTCOME_REFUSED } // The holder is absent before the JS runtime exists (and on a context @@ -66,7 +117,7 @@ class SelectableMarkdownModule(reactContext: ReactApplicationContext) : val contextHolder = reactApplicationContext.javaScriptContextHolder if (contextHolder == null) { Log.w(TAG, "install skipped: no JSI context holder (context not ready)") - return false + return OUTCOME_UNAVAILABLE } return try { @@ -85,9 +136,15 @@ class SelectableMarkdownModule(reactContext: ReactApplicationContext) : "install skipped: no JSI runtime pointer " + "(context tearing down, or running without a JSI runtime)", ) - false + OUTCOME_UNAVAILABLE + } else if (nativeInstall(runtimePointer)) { + OUTCOME_INSTALLED } else { - nativeInstall(runtimePointer) + // The C++ installer refused this runtime and has already + // written the reason to logcat (OnLoad.cpp). Nothing about + // asking again would change the answer for this runtime. + refused = true + OUTCOME_REFUSED } } } catch (error: Throwable) { @@ -95,9 +152,11 @@ class SelectableMarkdownModule(reactContext: ReactApplicationContext) : // case: the .so loaded but was built from a source tree whose // JNI symbol does not match this class. Caught with everything // else because no failure mode of an optional fast path is worth - // taking the app down for. + // taking the app down for. Permanent: the symbol will not appear + // without a rebuild. Log.w(TAG, "install failed: native install threw", error) - false + refused = true + OUTCOME_REFUSED } } @@ -116,6 +175,19 @@ class SelectableMarkdownModule(reactContext: ReactApplicationContext) : companion object { const val MODULE_NAME = "SelectableMarkdown" + /** + * The three answers `install()` can give. Read by + * src/engine/native/install.ts, which must spell them identically — + * the strings are the whole contract, and they are matched verbatim + * on the JS side (an unrecognised value there is read as + * `unavailable`, which is the safe direction: retry rather than give + * up). iOS returns the same three from + * platform/ios/SelectableMarkdownModule.mm. + */ + const val OUTCOME_INSTALLED = "installed" + const val OUTCOME_UNAVAILABLE = "unavailable" + const val OUTCOME_REFUSED = "refused" + private const val TAG = "SelectableMarkdown" private const val LIBRARY_NAME = "selectable-markdown" diff --git a/android/src/main/java/com/selectablemarkdown/SelectableRunHostView.kt b/android/src/main/java/com/selectablemarkdown/SelectableRunHostView.kt index cb01fcd..ed20729 100644 --- a/android/src/main/java/com/selectablemarkdown/SelectableRunHostView.kt +++ b/android/src/main/java/com/selectablemarkdown/SelectableRunHostView.kt @@ -4,12 +4,14 @@ import android.annotation.SuppressLint import android.os.Build import android.view.ActionMode import android.view.GestureDetector +import android.view.KeyEvent import android.view.Menu import android.view.MenuItem import android.view.MotionEvent import android.view.ViewGroup import android.widget.FrameLayout import android.widget.TextView +import androidx.core.view.ViewCompat import com.facebook.react.bridge.ReactContext import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap @@ -26,8 +28,10 @@ import com.facebook.react.uimanager.UIManagerHelper * JS contract (see docs/SELECTION.md): * props: `text` (projected run text), `attributes` (styled ranges over * that text), `pressables` (tappable ranges over that text), - * `selectable`, `selectionActions` (ordered action identifiers: - * "copy-text" | "copy-markdown") + * `selectable`, `exclusiveSelection` (whether this host takes part + * in the one-active-selection coordination), `selectionActions` + * (the ordered menu, one string per item: an action identifier, or + * `identifier + U+001F + title`) * events: `onSelectionAction({ start, end, action, selectedText })` — * UTF-16 code-unit offsets into the CURRENT `text`, * end-exclusive, clamped, start <= end; `action` names the menu @@ -35,12 +39,30 @@ import com.facebook.react.uimanager.UIManagerHelper * `onInlinePress({ start, end, pressableId })` — a single tap * landed inside one of `pressables`; same offset guarantees, and * `pressableId` is JS's identifier for the range, echoed verbatim. + * `onSelectionChange({ start, end })` — where the selection stands + * now, deduped; same offset guarantees except that an EMPTY range + * is a real payload and means "nothing is selected here". + * commands: `clearSelection()`, `setSelection(start, end)` — JS telling one + * mounted host what to select, in the same offsets the events + * report. Routed through the codegen'd ViewManager delegate. * * The system Copy item (android.R.id.copy) is never intercepted, replaced, * or reordered: stock plain-text copy keeps working with no JS involvement. * The custom items only EMIT the event — JS builds the payload and writes * the clipboard. * + * THE MENU'S STRINGS COME FROM JS WHEN JS SENDS THEM, and from this module's + * resources otherwise. An entry with no U+001F is a bare identifier, titled + * from `R.string.selectable_markdown_copy_text` / + * `..._copy_markdown` — which a host app overrides by declaring the same + * names, and translates with a `values-` folder. An entry that + * carries a title uses it verbatim, which is what lets one JS i18n call + * localise both platforms and what lets a consumer define items this file + * has never heard of. An identifier this view cannot title (unknown, and no + * title sent) is dropped rather than added as a blank menu item — the same + * forward-compatibility rule as before, now with the escape hatch that a + * title makes any identifier renderable. + * * `attributes` is what makes this host render markdown rather than a wall of * system text. Each entry is a range of `text` plus the parts of a text style * it changes, derived by JS from the same projection that produced `text`. @@ -60,51 +82,103 @@ import com.facebook.react.uimanager.UIManagerHelper * settled flowing block into one run keyed on its start offset, so when the * next block settles it lands in the same run: same key, same mounted view, * still `selectable=true`, longer text. Measured across the eight shipped - * fixtures, an Android-selectable run's text changes about four times per - * streamed message, and each one costs the user their selection and the open - * action mode. + * fixtures, an Android-selectable run's text changes four to ten times per + * streamed message — 47 changes across the eight, streamed in 5-character + * chunks under `presets.llmChat` — and each one costs the user their + * selection and the open action mode. * * Nothing here can copy the wrong markdown — the offsets are always read from * the current text — so this is a UX defect, not a correctness one. It is - * pre-existing and unfixed; docs/SELECTION.md ("Android — no preservation, and - * a known gap") carries the measurement and the two candidate fixes, both of - * which are larger than this file. + * pre-existing and unfixed; docs/SELECTION.md ("Android: no preservation, and + * a known gap") carries the per-fixture measurement and the candidate fixes, + * every one of which is larger than this file. * - * THE SAME OBJECT BACKS BOTH ARCHITECTURES. Nothing in this class is - * conditional on paper or Fabric: the view manager creates it either way, the - * props arrive either way (Fabric hands the Java ViewManager the raw props, so - * `RunAttributedText.parse` reads them unchanged), and both events go out - * through a dispatcher that resolves per architecture. What Fabric adds is - * recycling — see `prepareToRecycle`, which is a correctness requirement here - * and not hygiene. + * NOTHING HERE IS CONDITIONAL ON AN ARCHITECTURE, because there is only one + * left: the peer range starts at react-native 0.82. Fabric hands the Java + * ViewManager the raw props, so `RunAttributedText.parse` reads them + * unchanged, and events go out through `UIManagerHelper`'s dispatcher rather + * than a path of their own. What Fabric brings that the old architecture did + * not is recycling — see `prepareToRecycle`, which is a correctness + * requirement here and not hygiene. */ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { /** - * The host's text widget. A subclass for one behaviour the platform only + * The host's text widget. A subclass for the behaviours the platform only * offers to a TextView subclass, mirroring the iOS host: * `onSelectionChanged` feeds the one-active-selection coordination (see - * `liveHosts`), the only selection-change signal a TextView exposes. + * `activeHost`), the only selection-change signal a TextView exposes, and + * the three dispatch overrides below are the plumbing + * `ExploreByTouchHelper` documents as the caller's job. * * Everything else stays the stock widget — the decorator design's whole - * point — and the override degrades to `super` whenever it has nothing + * point — and every override degrades to `super` whenever it has nothing * to do. */ private inner class RunTextView(context: android.content.Context) : TextView(context) { override fun onSelectionChanged(selStart: Int, selEnd: Int) { super.onSelectionChanged(selStart, selEnd) - // The guard is also the constructor guard: TextView's own init - // reaches this override before the host's fields exist, always - // with an empty selection. - if (selEnd > selStart) { - clearOtherHostSelections() + // THE CONSTRUCTOR GUARD. TextView's own init reaches this override + // before the host's fields exist — `textView` itself is still + // null, so anything that reads it would NPE. It used to be + // implicit in `selEnd > selStart` (the selection is always empty + // that early); it has to be explicit now that an empty selection + // is something this view reports rather than ignores. + if (!readyForEvents) return + // Self first, coordination second, and the order is load-bearing: + // a hand-off must reach JS as "this run holds [4,9)" followed by + // "the other run holds nothing", which JS can drop as stale. + // Coordinating first would deliver a null and then the real + // selection — one visible toolbar flicker per hand-off. The iOS + // delegate is ordered the same way for the same reason. + emitSelectionChange() + // A host that opted out of exclusivity still REPORTS; it only + // skips the coordination. + if (selEnd > selStart && exclusiveSelection) { + becomeActiveSelectionHost() } } + + // The three feeds ExploreByTouchHelper cannot install for itself. + // Hover drives explore-by-touch (a finger dragged over the text with + // TalkBack on), keys drive arrow navigation between virtual views, + // and focus keeps the helper's idea of the focused node in step. All + // three are null-safe against construction order: this subclass is + // built while the host's own fields are still being initialised, and + // `accessibilityHelper` is the last of them. + override fun dispatchHoverEvent(event: MotionEvent): Boolean { + if (accessibilityHelper?.dispatchHoverEvent(event) == true) return true + return super.dispatchHoverEvent(event) + } + + override fun dispatchKeyEvent(event: KeyEvent): Boolean { + if (accessibilityHelper?.dispatchKeyEvent(event) == true) return true + return super.dispatchKeyEvent(event) + } + + override fun onFocusChanged( + focused: Boolean, + direction: Int, + previouslyFocusedRect: android.graphics.Rect?, + ) { + super.onFocusChanged(focused, direction, previouslyFocusedRect) + accessibilityHelper?.onFocusChanged(focused, direction, previouslyFocusedRect) + } } private val textView: TextView = RunTextView(context) + /** + * The screen-reader channel over `pressables` and the run's block roles — + * heading, list item, table cell; see `RunAccessibility.kt`. Nullable and + * assigned in `init` + * rather than initialised here, because `RunTextView` reads it from three + * dispatch overrides that the platform can in principle reach before this + * object finishes constructing. + */ + private var accessibilityHelper: RunAccessibilityHelper? = null + /** The three props the rendered text is built from, plus a dirty flag. * React Native delivers a whole prop batch and then calls * `onAfterUpdateTransaction`, so applying them there costs one setText @@ -119,6 +193,43 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { private var pendingEmbeds: RunEmbeds.Spec = RunEmbeds.Spec.EMPTY private var textDirty = false + /** The accessibility ranges' own dirty flag, and it cannot be folded into + * `textDirty`: they are derived from `text`, `attributes` AND + * `pressables`, and a pressables-only update deliberately never sets + * `textDirty` (see `setPressables`). Same purpose as `textDirty` though — + * under Fabric the whole prop map arrives on every commit, and this is + * what keeps a batch that changed none of the three from rebuilding the + * list. */ + private var accessibilityDirty = false + + /** False until `init` finishes. `RunTextView`'s own constructor reaches + * `onSelectionChanged` before this object's fields are assigned — before + * `textView` itself exists — so the emitter there has to know when it is + * safe to touch them. The JVM default of a Boolean field is false, which + * is what makes reading it from inside that constructor correct rather + * than merely lucky. */ + private var readyForEvents = false + + /** Whether this host takes part in the one-active-selection coordination; + * see `becomeActiveSelectionHost` and the `exclusiveSelection` prop. */ + private var exclusiveSelection = true + + /** The last range handed to `onSelectionChange`, so an unchanged selection + * is never re-announced. + * + * IT MATTERS BECAUSE `onSelectionChanged` IS NOISY: the platform calls it + * on every step of a handle drag and on every `Selection` write, and + * `commitProps` performs one of those on every commit that changes the + * text. An empty selection is normalised to (0, 0) before it lands here, + * so the two ways Android spells "nothing selected" — (-1, -1) and a + * collapsed cursor — dedupe against each other instead of alternating. + * + * Starting at (0, 0) means a host that has never held a selection emits + * nothing at all: an empty report is meaningful only as the END of a + * selection this host previously announced. */ + private var lastReportedStart = 0 + private var lastReportedEnd = 0 + /** Last rect reported per embedId, in dp — the dedupe that keeps * streaming appends past a settled embed from re-announcing it on every * snapshot. Values only ever compared against the next report. */ @@ -131,9 +242,24 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { private val decorationPath = android.graphics.Path() private val decorationRadii = FloatArray(8) - /** Configured menu actions, in prop order (identifiers from JS). */ - private var selectionActions: List = - listOf(ACTION_COPY_TEXT, ACTION_COPY_MARKDOWN) + /** Configured menu actions, in prop order: the identifier to report and + * the title to draw, both resolved when the prop arrived. Defaults to + * the built-in menu, matching the JS default. */ + private var selectionActions: List = defaultSelectionActions() + + /** The menu item ids added by the last `onPrepareActionMode`. + * + * Tracked rather than assumed because the menu is no longer a fixed two + * items: the prop can shrink, grow or be reordered between prepares, and + * removing exactly what was added is what leaves nothing behind. */ + private val addedMenuItemIds = ArrayList(2) + + /** Item id -> action identifier for the items currently on the menu. + * + * This map IS the "never intercept a system item" rule: only ids this + * class put on the menu are in it, so `onActionItemClicked` declines + * android.R.id.copy and every OEM addition by finding nothing. */ + private val menuItemActions = HashMap() /** Tappable ranges over the text, parsed from the `pressables` prop. * Empty whenever JS has no listener, which is what keeps every touch @@ -149,6 +275,14 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { * ClickableSpan + LinkMovementMethod, which take over the TextView's * movement/touch handling and are a known source of selection breakage on * exactly the widget this class exists to keep stock. + * + * IT IS NOT THE ONLY WAY IN ANY MORE, and it could not be: a screen + * reader activates a node with ACTION_CLICK through the accessibility + * API, never by injecting a touch stream, so while this detector was the + * only path a link inside a run was unreachable with TalkBack on — and + * unannounced. `accessibilityHelper` adds that path (RunAccessibility.kt) + * without giving up anything above: it installs no movement method, and + * both routes end in the same `emitInlinePress`. */ private val inlineTapDetector = GestureDetector( context, @@ -179,33 +313,36 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { // Remove-first, then re-add per the current prop: repeated // prepare calls (OEM skins invoke it more than once) and prop // updates both converge on the same menu with no duplicates. - // System items (android.R.id.copy and friends) stay exactly - // where the platform put them. - menu.removeItem(ITEM_ID_COPY_TEXT) - menu.removeItem(ITEM_ID_COPY_MARKDOWN) + // What gets removed is exactly what was added last time, not a + // fixed pair — the list is consumer-sized now. System items + // (android.R.id.copy and friends) stay exactly where the + // platform put them. + for (index in addedMenuItemIds.indices) { + menu.removeItem(addedMenuItemIds[index]) + } + addedMenuItemIds.clear() + menuItemActions.clear() var order = Menu.CATEGORY_SECONDARY for (action in selectionActions) { - val itemId = when (action) { - ACTION_COPY_TEXT -> ITEM_ID_COPY_TEXT - ACTION_COPY_MARKDOWN -> ITEM_ID_COPY_MARKDOWN - // Forward-compat: identifiers this binary does not know - // are ignored, never rendered as dead items. - else -> continue - } - menu.add(Menu.NONE, itemId, order, titleFor(action)) + // Sequential from the base, so the default two-item menu + // still gets the same two ids it always had. Every entry in + // `selectionActions` is renderable — an identifier this + // binary cannot title was already dropped by + // `parseSelectionAction` — so there is nothing to skip here. + val itemId = ITEM_ID_BASE + addedMenuItemIds.size + menu.add(Menu.NONE, itemId, order, action.title) + addedMenuItemIds.add(itemId) + menuItemActions[itemId] = action.id order += 1 } return true } override fun onActionItemClicked(mode: ActionMode, item: MenuItem): Boolean { - val action = when (item.itemId) { - ITEM_ID_COPY_TEXT -> ACTION_COPY_TEXT - ITEM_ID_COPY_MARKDOWN -> ACTION_COPY_MARKDOWN - // Never intercept system items (android.R.id.copy etc.); - // plain copy must keep its stock behavior. - else -> return false - } + // Never intercept system items (android.R.id.copy etc.); plain + // copy must keep its stock behavior. Only the ids this callback + // added are in the map, so anything else declines by missing. + val action = menuItemActions[item.itemId] ?: return false emitSelectionAction(action) mode.finish() return true @@ -221,11 +358,12 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { init { textView.setTextIsSelectable(true) // Every knob that appears on both sides of the measure/draw boundary — - // text size, font padding, line spacing, line breaking — is set here + // text size, font padding, line spacing, line breaking, paragraph + // direction, fallback line spacing — is set here // and only here. RunTextMeasure owns them because the two things that // must agree about them are this TextView and the StaticLayout built - // by the measure path (paper's shadow node, or the C++ shadow node - // through SelectableRunHostViewManager.measure). Setting any of them + // by the measure path (the C++ shadow node, through + // SelectableRunHostViewManager.measure). Setting any of them // directly on this view again would be the drift that file exists to // prevent, and the symptom is text clipped at the bottom of a run. RunTextMeasure.configureTextView(textView) @@ -236,19 +374,42 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { // builds; markdown runs need none of its smart-selection output. textView.textClassifier = android.view.textclassifier.TextClassifier.NO_OP } + // The screen-reader channel, installed once and for the life of the + // view: it offers no node provider at all while the run has no links + // and no block roles + // (RunAccessibilityHelper.getAccessibilityNodeProvider), so there is + // nothing to attach and detach as props change. + // + // ExploreByTouchHelper's constructor forces `focusable` on and lifts + // importantForAccessibility from AUTO to YES; both are restored here, + // exactly as React Native's own ReactAccessibilityDelegate restores + // them (ReactAccessibilityDelegate.java:403-407), so a run keeps the + // focus behaviour and the announcement coalescing the stock widget + // had. `setTextIsSelectable` above owns `focusable` on this widget, + // and `setSelectable` keeps owning it afterwards. + val focusableBefore = textView.isFocusable + val importanceBefore = textView.importantForAccessibility + val helper = RunAccessibilityHelper(textView) { pressable -> emitInlinePress(pressable) } + textView.isFocusable = focusableBefore + textView.importantForAccessibility = importanceBefore + // ViewCompat, not View#setAccessibilityDelegate: the helper is an + // AccessibilityDelegateCompat, which the framework setter does not + // take. + ViewCompat.setAccessibilityDelegate(textView, helper) + accessibilityHelper = helper addView( textView, LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT) ) - // One-active-selection coordination — see `liveHosts`. Registered - // last, so a host in the registry always has fully initialised - // fields. - liveHosts.add(this) // A ViewGroup skips its own onDraw by default; block chrome (boxes, // rules) is painted there, behind the child TextView — which is the // stacking the design needs, since the platform draws the selection // highlight inside the TextView, above whatever this layer painted. setWillNotDraw(false) + // LAST LINE OF `init`, deliberately: everything `onSelectionChanged` + // touches — `textView`, the dedupe fields, `exclusiveSelection` — is + // assigned by now, so the guard in that override can stop refusing. + readyForEvents = true } // ---- Props ------------------------------------------------------------- @@ -257,6 +418,7 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { if (value == pendingText) return pendingText = value textDirty = true + accessibilityDirty = true } /** @@ -266,9 +428,10 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { * dismisses the ActionMode even when the new value is character-identical * to the old one. * - * On the old architecture an unguarded setter was harmless: paper sends - * only the props JS diffed as changed, so this ran when `attributes` - * actually changed. Fabric does not diff. `FabricMountingManager::getProps` + * An unguarded setter used to be harmless, because the old architecture + * sent only the props JS had diffed as changed, so this ran when + * `attributes` actually changed. Fabric does not diff, and Fabric is the + * only architecture this package supports. `FabricMountingManager::getProps` * returns `newShadowView.props->rawProps` whole * (ReactAndroid/src/main/jni/react/fabric/FabricMountingManager.cpp:222-226), * `SurfaceMountingManager` wraps that entire map, and @@ -288,6 +451,9 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { if (spec.attributes == pendingAttributes.attributes) return pendingAttributes = spec textDirty = true + // Headings, list items and table cells are read back out of these + // ranges (RunAccessibility.kt). + accessibilityDirty = true } /** @@ -321,6 +487,18 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { /** Applied once per prop batch, from the view manager. */ fun commitProps() { + // Ahead of the early return below, because the accessibility ranges + // have their own dirty flag: `pressables` changing on its own is the + // case that must not be missed, and it deliberately leaves `text` + // alone. Offsets only — the node bounds are read from the layout at + // the moment a screen reader asks for them, so this does not care + // that the text below has not been installed yet. + if (accessibilityDirty) { + accessibilityDirty = false + accessibilityHelper?.setNodes( + RunAccessibility.resolve(pendingText, pendingAttributes, pressables) + ) + } if (!textDirty) return textDirty = false // No selection preservation, and the class comment is honest about @@ -349,6 +527,26 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { // the previous run's base size. The measure paths derive the same // value from the same two props (RunTextMeasure.baseTextSizeSp). RunTextMeasure.updateTextViewBaseSize(textView, pendingText, pendingAttributes) + // And the room a box at the very edge of this run needs, as the child + // TextView's vertical padding. + // + // WHY PADDING RATHER THAN A LAYOUT OFFSET. `totalPaddingTop` is + // already what every text-to-view coordinate conversion in this file + // goes through — the box and rule geometry in `onDraw`, the embed + // rects, the pressable hit test, the screen reader's node bounds — so + // pushing the text down this way moves all of them together and none + // of them has to learn why. The measure path reserved exactly these + // two values — the same `RunDecorations.edgePaddingPx` call, added to + // the height in `RunTextMeasure.measure`, so the rounding to whole + // pixels `setPadding` needs happens once and both sides spend the + // identical integers. The taller view Fabric framed is therefore the + // room this padding fills; without it the padding would grow the + // TextView past the host and clip the last line instead. + // + // Zero for the ordinary run — no box at either edge — in which case + // this is the `setPadding(0, 0, 0, 0)` the widget already had. + val edge = RunDecorations.edgePaddingPx(pendingDecorations, pendingText.length) + textView.setPadding(0, edge.top, 0, edge.bottom) textView.text = RunLayoutCache.styledText( RunLayoutCache.key(pendingText, pendingAttributes, pendingDecorations, pendingEmbeds) ) @@ -356,6 +554,13 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { // own display list is stale the moment the text moves — and a child // invalidation alone does not rebuild the parent's. invalidate() + // The swap above dropped whatever was selected, and JS has to be told + // — a toolbar over a selection that no longer exists is exactly the + // artifact this event was added to remove. Called rather than left to + // `onSelectionChanged`, because whether `TextView#setText` re-notifies + // for the NEW text is a version-dependent detail of the widget; the + // dedupe makes a duplicate call free. + emitSelectionChange() // Embed rects are read off the TextView's Layout, which does not // exist until the layout pass the setText above requested. onLayout // is the primary report point; the post is the belt for commits the @@ -370,6 +575,91 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { textView.setTextIsSelectable(value) } + /** + * The `exclusiveSelection` prop: whether this host takes part in the + * one-active-selection coordination (`becomeActiveSelectionHost`). + * Defaults to true, which is what every host did before the prop existed. + * + * FALSE OPTS OUT IN BOTH DIRECTIONS. This host neither clears the previous + * owner nor takes the slot, so it cannot erase another host's selection + * and — because it is never the recorded owner — no other host can erase + * its. An opt-out that only stopped the clearing would be useless: the + * first selection would still die the moment a second one began. + * + * WHAT `false` BUYS, AND WHAT IT DOES NOT. It buys several simultaneous + * `Selection` spans that survive each other, each reported by its own + * host through `onSelectionChange` — which is what makes "select in A, + * select in B, merge the two payloads" reachable at all. It does NOT buy + * several visible highlights, and it buys at most one action mode. A + * `TextView` draws a selection highlight only while `isFocused() || + * isPressed()` (TextView#getUpdatedHighlightPath — the same fact + * `setSelection` is built around), and only one view has focus, so the + * earlier selections are live and invisible: the user sees the highlight + * move to whichever run they touched last. Anything built on this has to + * give its own feedback for the spans it is accumulating; the platform + * will not. + * + * The baton is handed back here rather than at the next selection change, + * because a host that opted out while holding it would otherwise be + * cleared once more by the next selection elsewhere, after it had already + * stopped participating. + */ + fun setExclusiveSelection(value: Boolean) { + if (value == exclusiveSelection) return + exclusiveSelection = value + if (!value && activeHost?.get() === this) { + activeHost = null + } + } + + // ---- Commands ----------------------------------------------------------- + + /** + * The `setSelection` command: select `[start, end)` of the current text, + * UTF-16 offsets, end-exclusive — the same unit and the same clamping + * discipline as every event this view emits, because JS computed these + * offsets against text that may have moved on by a frame. + * + * FOCUS IS TAKEN, AND IT HAS TO BE. `TextView` draws a selection highlight + * only while `isFocused() || isPressed()` (TextView#getUpdatedHighlightPath), + * so setting the `Selection` spans alone would be an invisible selection: + * present in `selectionStart`/`End`, reported to JS as real, and absent + * from the screen. The consequence is worth stating rather than hiding — + * a scrolling ancestor may respond to a focus change by scrolling this run + * into view. This command does not scroll; the platform's focus handling + * may. + * + * No action mode is started. A programmatic selection shows the range and + * its handles; the menu belongs to the user's gesture, and iOS behaves the + * same way. + * + * A run that is not selectable takes nothing — the platform's selection UI + * is off there (the streaming tail under the Android tail policy, or an + * explicit `selectable={false}`), so a selection would be state nobody can + * see or dismiss. + * + * A RANGE THAT CLAMPS TO EMPTY IS A NO-OP AND LEAVES ANY EXISTING + * SELECTION ALONE. It used to clear, which made the command destructive + * in the one case it is least sure of itself: the offsets were computed + * against text that may have moved on by a frame, so an empty clamp means + * "I raced a swap", not "the app asked for nothing" — and a user mid-sweep + * in this run would have lost their selection to a command aimed at text + * that is no longer here. Clearing is what `clearSelection` is for. + * Nothing changes, so nothing is emitted: `onSelectionChange` keeps + * describing the selection this run actually has. iOS follows the same + * rule, for the same reason (`SelectableRunHostView.setSelection`). + */ + fun setSelection(start: Int, end: Int) { + if (!textView.isTextSelectable) return + val spannable = textView.text as? android.text.Spannable ?: return + val length = spannable.length + val low = minOf(start, end).coerceIn(0, length) + val high = maxOf(start, end).coerceIn(0, length) + if (high <= low) return + textView.requestFocus() + android.text.Selection.setSelection(spannable, low, high) + } + /** * Deliberately independent of `text`/`attributes` and of `textDirty`: * what is tappable and what is drawn are separate channels, so a @@ -378,43 +668,199 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { * discipline exists). */ fun setPressables(value: List) { + if (value == pressables) return pressables = value + // The screen reader's link nodes are these same ranges, so they are + // rebuilt with them — in `commitProps`, once for the whole batch, + // like everything else derived from props. + accessibilityDirty = true } + /** + * The `selectionActions` prop: one string per menu item, either an + * action identifier or `identifier + U+001F + title`, in menu order. + * + * Resolved to titles HERE rather than in `onPrepareActionMode`, for the + * same reason `pressables` is parsed on arrival: prepare runs inside a + * UIKit-equivalent callback that OEM skins invoke repeatedly while a + * selection is live, and it must not do string work or resource lookups + * per invocation. The comparison after resolving is also what keeps a + * prop batch that re-sends an identical list from invalidating an open + * action mode. + */ fun setSelectionActions(actions: List) { - if (actions == selectionActions) { + val resolved = ArrayList(actions.size) + for (index in actions.indices) { + val action = parseSelectionAction(actions[index]) + if (action != null) { + resolved.add(action) + } + } + if (resolved == selectionActions) { return } - selectionActions = actions + selectionActions = resolved // An open menu rebuilds through onPrepareActionMode. activeActionMode?.invalidate() } + /** + * One `selectionActions` entry, split the way + * `src/view/selectionActions.ts` packs it. + * + * The split is at the FIRST U+001F and everything after it is the title + * verbatim, so a title that contains one survives; an identifier could + * not, which is why the encoder refuses to send one. Returns null — the + * item is dropped, never added blank — for an empty identifier, and for + * an identifier that has no title from JS and none of this library's + * own. That last case is the forward-compatibility rule: a newer JS + * bundle naming an action this binary predates is ignored. + */ + private fun parseSelectionAction(encoded: String): ResolvedAction? { + val separator = encoded.indexOf(ACTION_TITLE_SEPARATOR) + val id = if (separator < 0) encoded else encoded.substring(0, separator) + if (id.isEmpty()) { + return null + } + val sent = if (separator < 0) "" else encoded.substring(separator + 1) + val title = if (sent.isNotEmpty()) sent else defaultTitleFor(id) + if (title == null) { + return null + } + return ResolvedAction(id, title) + } + + /** + * This library's own title for an identifier it implements, or null for + * one it does not know. + * + * A RESOURCE AND NOT A LITERAL, so the two built-in items can be + * translated and reworded without a fork: a host app declares the same + * string names in its own `res/values/strings.xml` to override them (an + * application resource wins over a library's) and adds + * `res/values-/` to translate them. It is the Android + * counterpart of the iOS host's `NSLocalizedString` lookup against + * `Bundle.main`. Sending a `title` from JS overrides both at once and is + * the only path that reaches a consumer-defined identifier. + */ + private fun defaultTitleFor(id: String): String? = when (id) { + ACTION_COPY_TEXT -> context.getString(R.string.selectable_markdown_copy_text) + ACTION_COPY_MARKDOWN -> context.getString(R.string.selectable_markdown_copy_markdown) + else -> null + } + + /** The menu before any prop arrives, and after a prop reset: literally + * what the JS default (`DEFAULT_SELECTION_ACTIONS`) encodes to — the two + * bare identifiers, run through the same parse as any other entry, so + * there is one place where a built-in item gets its title. */ + private fun defaultSelectionActions(): List = listOfNotNull( + parseSelectionAction(ACTION_COPY_TEXT), + parseSelectionAction(ACTION_COPY_MARKDOWN), + ) + // ---- Selection coordination --------------------------------------------- /** * One active selection across the document: Android never clears one * TextView's selection because another began one, so a transcript of - * per-run hosts could show two highlights at once — only the newest with - * a live action mode. Called from `RunTextView.onSelectionChanged` the - * moment a non-empty selection lands here. Recursion-safe: clearing - * another host fires its `onSelectionChanged` with an empty selection, - * which returns at the guard. + * per-run hosts would otherwise keep several live `Selection` spans at + * once. Not several highlights — a `TextView` draws one only while it is + * focused or pressed, and only one view has focus — so the older span + * goes invisible the instant the newer one begins, while remaining a real + * range its host has already reported to JS. That undrawn, undismissable + * state is what this removes. Called from + * `RunTextView.onSelectionChanged` the moment a non-empty selection lands + * here. + * + * THE PREDECESSOR IS THE ONLY HOST THAT CAN BE HOLDING ONE, which is what + * makes this O(1). The invariant is maintained by this method itself: + * every non-empty selection passes through here and clears the one before + * it, so at most one host in the process has a selection, and + * `activeHost` is it. This used to sweep a weak registry of every live + * host on every callback — an allocation and a walk over every mounted + * run, per frame, for the whole of a selection-handle drag. + * + * Recursion-safe: clearing the predecessor fires its + * `onSelectionChanged` with an empty selection, which returns at the + * guard there. The reference is weak, so an unmounted predecessor is + * simply gone by the time it would have been cleared, and nothing here + * keeps a view (or its ReactContext) alive. */ - private fun clearOtherHostSelections() { - for (host in liveHosts.toList()) { - if (host === this) continue - val other = host.textView.text as? android.text.Spannable ?: continue - if (host.textView.selectionEnd > host.textView.selectionStart) { - android.text.Selection.removeSelection(other) - host.activeActionMode?.finish() - } - } + private fun becomeActiveSelectionHost() { + val previous = activeHost?.get() + if (previous === this) return + activeHost = java.lang.ref.WeakReference(this) + previous?.clearSelection() + } + + /** + * Drop this host's selection and close any menu over it. A no-op when + * nothing is selected. + * + * ONE METHOD FOR TWO CALLERS, deliberately: the `clearSelection` command + * from JS and the coordination clearing the previous selection owner mean + * exactly the same thing, and two definitions of "this run has no + * selection" would be two chances to leave an orphaned action mode + * behind. Order matters the same way it does in `prepareToRecycle`: the + * selection goes first, because finishing the mode is what the platform + * does in response. + * + * The report is left to the `onSelectionChanged` the removal triggers. + */ + fun clearSelection() { + if (textView.selectionEnd <= textView.selectionStart) return + val spannable = textView.text as? android.text.Spannable ?: return + android.text.Selection.removeSelection(spannable) + activeActionMode?.finish() } // ---- Event emission ---------------------------------------------------- + /** + * Report where the selection stands, deduped against the last report. + * + * The clamp is `emitSelectionAction`'s, minus its "never empty" rule: an + * empty range is the whole reason this event exists, so it is emitted — + * once — when it follows a non-empty one. Android spells "nothing + * selected" two ways, (-1, -1) and a collapsed offset, and both normalise + * to (0, 0) here so they dedupe against each other rather than + * alternating. + * + * Unlike iOS this needs no forced re-announcement after a text swap: on + * this platform `setText` DROPS the selection rather than clamping it + * through, so the swap always produces a genuine range change for the + * dedupe to notice. + */ + private fun emitSelectionChange() { + val length = textView.text?.length ?: 0 + val rawStart = textView.selectionStart + val rawEnd = textView.selectionEnd + var start = 0 + var end = 0 + if (rawStart >= 0 && rawEnd >= 0) { + start = minOf(rawStart, rawEnd).coerceIn(0, length) + end = maxOf(rawStart, rawEnd).coerceIn(0, length) + if (end <= start) { + start = 0 + end = 0 + } + } + if (start == lastReportedStart && end == lastReportedEnd) return + lastReportedStart = start + lastReportedEnd = end + + // One dispatch for both architectures, exactly as emitSelectionAction + // below explains. Recorded above before the dispatcher is resolved, so + // a report that cannot be delivered (a detached view) does not leave + // the dedupe claiming the previous range is still current. + val reactContext = context as ReactContext + val dispatcher = UIManagerHelper.getEventDispatcherForReactTag(reactContext, id) ?: return + dispatcher.dispatchEvent( + SelectionChangeEvent(UIManagerHelper.getSurfaceId(this), id, start, end) + ) + } + private fun emitSelectionAction(action: String) { val length = textView.text?.length ?: 0 // TextView reports UTF-16 offsets; they may be -1 (no selection) or @@ -476,8 +922,10 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { if (x < layout.getLineLeft(line) || x > layout.getLineRight(line)) return null val offset = layout.getOffsetForHorizontal(line, x) // Half-open containment, matching how LinkMovementMethod queries - // spans at an insertion offset: ranges never overlap (links cannot - // nest), so the first hit is the only hit. + // spans at an insertion offset: ranges never overlap — JS's + // resolveRunPressables drops any range starting inside one it already + // kept, link marks themselves do nest — so the first hit is the only + // hit. return pressables.firstOrNull { offset >= it.start && offset < it.end } } @@ -526,11 +974,18 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { * * GEOMETRY IS ANCHORED ON THE LINE, NOT THE BASELINE. The reservation is * ascent-shaped in `RunEmbedSpan`, but the final line extents belong to - * the embed-height `RunLineHeightSpan` JS sends over the same character, - * and its surplus branch re-centres the extra room around the baseline — - * so `getLineBaseline - height` can point above the line's top. The top - * of the placeholder's line IS the top of the reserved band, whatever - * the baseline did. + * the embed-height `RunLineHeightSpan` that `RunEmbeds.applyLineHeights` + * sets over the same character, and its surplus branch re-centres the + * extra room around the baseline — so `getLineBaseline - height` can + * point above the line's top. The top of the placeholder's line IS the + * top of the reserved band, whatever the baseline did. + * + * REPORTING THE DECLARED SIZE IS ONLY HONEST BECAUSE THE RESERVATION IS + * IN THE SAME UNIT. The rect below echoes `widthDp`/`heightDp` back + * unconverted, and the box the span holds open is those same numbers + * through `toPixelFromDIP` — including the line height, which is why that + * one is decoded in DIP rather than in the SP every other line height on + * the wire uses. * * The horizontal edge takes the smaller of the two `getPrimaryHorizontal` * answers: in an RTL paragraph the placeholder's leading edge is its @@ -552,7 +1007,8 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { val textTop = (textView.top + textView.totalPaddingTop - textView.scrollY).toFloat() for (embed in embeds) { - // The same guards `RunEmbeds.applySpans` applied to the string: + // The same guards `RunEmbeds.forEachReserved` applied when the + // string was built (both halves of the reservation go through it): // an entry that reserved nothing must report nothing. if (embed.start >= length || embed.end > length) continue if (text[embed.start] != RunEmbeds.PLACEHOLDER) continue @@ -662,9 +1118,23 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { val textTop = textView.top.toFloat() + textView.totalPaddingTop val firstLine = layout.getLineForOffset(start) val lastLine = layout.getLineForOffset(end - 1) - // Padding extends into the blank separator lines around the block; - // the clamp keeps a box at the run's very edge inside this view - // instead of painted over a neighbour (the parent clips anyway). + // Padding is normally painted into the blank line the projection's + // '\n\n' block separator leaves around the block, so it costs no + // height. At the EDGE of a run there is no such line, and the room + // comes from `RunDecorations.edgePaddingPx` instead: the measure path + // added it to this host's height and `commitProps` set it as the + // child TextView's padding, so `textTop` for a box starting at offset + // 0 is already `paddingTop` or more, and this view's `height` for one + // ending at the last character is already `paddingBottom` or more + // past the last line's bottom. + // + // THE CLAMPS THEREFORE NO LONGER BITE FOR A WELL-FORMED DECORATION, + // and they stay because that is not the only kind that can arrive: an + // entry from a newer JS bundle can name a padding larger than the + // room JS asked to reserve, and chrome painted over a neighbouring + // view is worse than chrome drawn a pixel short. They were a bug when + // they were the ONLY thing between a trailing table and a border on + // its own last baseline. val top = (textTop + layout.getLineTop(firstLine) - PixelUtil.toPixelFromDIP(decoration.paddingTop)).coerceAtLeast(0f) val bottom = (textTop + layout.getLineBottom(lastLine) + @@ -790,6 +1260,12 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { pendingText = "" pendingAttributes = RunAttributedText.Spec.EMPTY pendingDecorations = RunDecorations.Spec.EMPTY + // The run-edge room the previous run's box asked for, which is + // geometry and not a prop: a recycled host that kept it would offset + // the next run's text by a padding that run never reserved, and the + // measured height it was framed at would not include. `commitProps` + // recomputes it from the decorations the next run brings. + textView.setPadding(0, 0, 0, 0) // Embeds and their report ledger go together: a recycled host that // kept either could report the PREVIOUS run's rects against the next // run's embedIds — the embed cousin of the stale-selection failure @@ -797,12 +1273,42 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { pendingEmbeds = RunEmbeds.Spec.EMPTY lastEmbedRects.clear() textDirty = false - selectionActions = listOf(ACTION_COPY_TEXT, ACTION_COPY_MARKDOWN) + selectionActions = defaultSelectionActions() + // The menu the finished action mode above was showing is gone, so + // these only ever describe items that no longer exist. Cleared + // anyway: a recycled host must not carry a mapping from the previous + // run's menu ids to the previous run's action identifiers. + addedMenuItemIds.clear() + menuItemActions.clear() // A fresh host has no tappable ranges; a recycled one keeping the // previous run's would turn arbitrary spots of the next run's prose // into links — the pressable cousin of the stale-selection failure // this method exists to prevent. pressables = emptyList() + // And the screen reader's view of them, for the same reason: a + // virtual link node left over from the previous run would offer a + // TalkBack user an activation on text that is gone. + accessibilityDirty = false + accessibilityHelper?.setNodes(emptyList()) + // Hand back the one-active-selection baton. The `textView.text = ""` + // above already dropped this host's selection, so leaving it on + // record as the document's selection owner would only cost the next + // selecting host a no-op call — but the invariant is worth keeping + // true rather than merely harmless. + if (activeHost?.get() === this) { + activeHost = null + } + // Selection-report history, like `lastEmbedRects` above: it describes + // the previous run's offsets. A recycled host that kept it could + // suppress the first real report of the NEXT run's selection as a + // duplicate — the same stale-state failure class, one channel over. + lastReportedStart = 0 + lastReportedEnd = 0 + // And the exclusivity policy goes back to the default a freshly + // constructed host has, for the same reason `selectable` does: it is + // the safe value (coordinate), and the prop is re-sent by the batch + // that follows on every run where it matters. + exclusiveSelection = true } // ---- Lifecycle & platform-bug containment ------------------------------- @@ -823,7 +1329,8 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { // `removeClippedSubviews` detaches and re-attaches the very same view // as it scrolls, and without this the callback uninstalled above was // gone for good: the run stayed selectable but permanently lost - // "Copy Text" and "Copy Markdown", with no error to notice. + // every item `selectionActions` asked for — "Copy Text" and "Copy + // Markdown" by default — with no error to notice. textView.customSelectionActionModeCallback = selectionCallback } @@ -866,19 +1373,34 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { */ internal data class Pressable(val start: Int, val end: Int, val id: Int) + /** + * One resolved menu item: the identifier reported in + * `onSelectionAction`, and the string drawn on the item. A data class + * because `setSelectionActions` compares whole lists by value to decide + * whether an open action mode has to be invalidated. + */ + internal data class ResolvedAction(val id: String, val title: String) + companion object { const val ACTION_COPY_TEXT = "copy-text" const val ACTION_COPY_MARKDOWN = "copy-markdown" /** - * Every live host, weakly, for one-active-selection coordination — - * the Android twin of the iOS `liveHosts` NSHashTable. A weak set, - * so unmounted hosts fall out on their own; touched from the main - * thread only (selection changes and clears are both UI-thread - * events), so no synchronization is needed. + * The one host in the process that currently holds a selection, or + * null — the whole of the one-active-selection coordination (see + * `becomeActiveSelectionHost`). + * + * Weak, so an unmounted host is collected rather than pinned here + * with the ReactContext behind it; touched from the main thread only + * (selection changes and clears are both UI-thread events), so no + * synchronization is needed. Process-wide, which is the documented + * behaviour (docs/SELECTION.md, "Selections never span hosts") and + * also its limitation: two unrelated `` trees in + * a split view clear each other unless one of them sets + * `exclusiveSelection={false}` (see `setExclusiveSelection`), which + * is the only opt-out and is per-host, not per-tree. */ - private val liveHosts: MutableSet = - java.util.Collections.newSetFromMap(java.util.WeakHashMap()) + private var activeHost: java.lang.ref.WeakReference? = null /** * Parses the `pressables` prop, with `RunAttributedText.parse`'s @@ -911,15 +1433,25 @@ class SelectableRunHostView(context: ReactContext) : FrameLayout(context) { null } - /** Distinctive high item ids ("SM" + index). They can never equal - * the small sequential ids OEM menus and ACTION_PROCESS_TEXT items - * use, nor any android.R.id constant (those live in 0x0102xxxx). */ - private const val ITEM_ID_COPY_TEXT = 0x53_4D_01 - private const val ITEM_ID_COPY_MARKDOWN = 0x53_4D_02 + /** + * First of the distinctive high item ids ("SM" + index); the nth + * rendered custom item gets `ITEM_ID_BASE + n`, so the default + * two-item menu keeps the exact ids it always had. + * + * They can never equal the small sequential ids OEM menus and + * ACTION_PROCESS_TEXT items use, nor any android.R.id constant + * (those live in 0x0102xxxx) — and a list long enough to walk out of + * the 0x534Dxx band walks into 0x534Exx, which collides with neither. + */ + private const val ITEM_ID_BASE = 0x53_4D_01 - private fun titleFor(action: String): String = when (action) { - ACTION_COPY_TEXT -> "Copy Text" - else -> "Copy Markdown" - } + /** + * The character `src/view/selectionActions.ts` packs an item's + * identifier and title around: U+001F INFORMATION SEPARATOR ONE, + * chosen because no menu title can legitimately contain it and no + * identifier in this library uses it. An entry without it is a bare + * identifier, which is the pre-title wire format unchanged. + */ + private const val ACTION_TITLE_SEPARATOR = '\u001F' } } diff --git a/android/src/main/java/com/selectablemarkdown/SelectableRunHostViewManager.kt b/android/src/main/java/com/selectablemarkdown/SelectableRunHostViewManager.kt index 0644d14..fcfb16e 100644 --- a/android/src/main/java/com/selectablemarkdown/SelectableRunHostViewManager.kt +++ b/android/src/main/java/com/selectablemarkdown/SelectableRunHostViewManager.kt @@ -16,16 +16,17 @@ import com.facebook.yoga.YogaMeasureMode import com.facebook.yoga.YogaMeasureOutput /** - * Manager for the `SelectableRunHost` component, on **both** architectures. + * Manager for the `SelectableRunHost` component. * * The name must match the string in `codegenNativeComponent('SelectableRunHost')` * (src/view/SelectableRunHostNativeComponent.ts): it is what * `FabricUIManager.measure` and the mounting layer route on, and what the C++ * component name resolves to. * - * FABRIC ONLY. The package's peer range starts at react-native 0.82, where - * the old architecture cannot be installed, so the Kotlin measuring shadow - * node this manager used to build and type is gone. The shadow node for + * FABRIC ONLY, AND THERE IS NO ARCHITECTURE GATE LEFT ANYWHERE IN THIS + * MODULE. The package's peer range starts at react-native 0.82, where the old + * architecture cannot be installed, so the Kotlin measuring shadow node this + * manager used to build and type is gone. The shadow node for * `SelectableRunHost` is C++ — `RNSMRunHostShadowNode` in platform/fabric, * registered through the component descriptor the app's generated * autolinking.cpp instantiates. Measurement arrives through `measure` below, @@ -39,7 +40,7 @@ import com.facebook.yoga.YogaMeasureOutput * * IMPLEMENTING THE GENERATED INTERFACE IS THE POINT OF THE CODEGEN SETUP. * `SelectableRunHostManagerInterface` is generated from the TypeScript spec by - * React Native's Gradle plugin, for libraries regardless of architecture + * React Native's Gradle plugin, for every library that applies it * (ReactPlugin.kt:91-93 with an onlyIf that passes for every * com.android.library). It is the only compile-time link between the spec and * these setters: rename a prop in TypeScript and this file stops compiling, @@ -56,11 +57,11 @@ class SelectableRunHostViewManager : * * With a delegate present, `ViewManager.updateProperties` routes each prop * through `delegate.setProperty` instead of reflecting over @ReactProp - * (ViewManager.java:82-89). That is the same set of props either way — the - * generated switch calls the interface methods this class implements, and - * its default branch hands the base view props to BaseViewManagerDelegate — - * so paper behaviour is unchanged and Fabric gets the generated path it - * expects. + * (ViewManager.java:82-89). That is the same set of props reflection would + * have found — the generated switch calls the interface methods this class + * implements, and its default branch hands the base view props to + * BaseViewManagerDelegate — so nothing is lost by taking the generated + * path, and taking it is what makes the spec a compile-time contract. */ private val delegate: ViewManagerDelegate = SelectableRunHostManagerDelegate(this) @@ -70,11 +71,16 @@ class SelectableRunHostViewManager : override fun getName(): String = COMPONENT_NAME override fun createViewInstance(reactContext: ThemedReactContext): SelectableRunHostView { - // Earliest point this component holds a Context: arm the cache's - // trim-memory reclamation (idempotent — one registration per process, - // on the application context, so it survives ReactInstance teardown - // the same way the cache singleton does). + // Two process-wide singletons this component owns are armed from the + // Contexts it is handed, both idempotent and both registered on the + // APPLICATION context so they survive ReactInstance teardown the way + // the singletons themselves do: the layout cache's trim-memory + // reclamation, and the font resolver's asset table. `measure` arms the + // font resolver too, because under Fabric it runs before anything + // mounts — and a measurement taken before families resolve would be + // cached under a key that cannot tell the difference. RunLayoutCache.installTrimHook(reactContext) + RunTypefaces.install(reactContext) return SelectableRunHostView(reactContext as ReactContext) } @@ -133,6 +139,44 @@ class SelectableRunHostViewManager : view.setSelectable(selectable) } + /** + * Whether this host takes part in the process-wide one-active-selection + * coordination. `defaultBoolean = true` matches the spec's + * `WithDefault` and the view's own field, so a prop reset + * restores coordinating — the safe direction, since the failure of the + * other one is two live highlights with only the newest carrying handles. + */ + @ReactProp(name = "exclusiveSelection", defaultBoolean = true) + override fun setExclusiveSelection(view: SelectableRunHostView, exclusive: Boolean) { + view.setExclusiveSelection(exclusive) + } + + /** + * The imperative half of the contract: the two codegen commands. + * + * THERE IS NO `receiveCommand` OVERRIDE HERE, AND THERE MUST NOT BE ONE. + * `ViewManager.receiveCommand(root, commandId, args)` already asks + * `getDelegate()` and forwards to it (ViewManager.java:296-301), and this + * manager returns the codegen'd `SelectableRunHostManagerDelegate` from + * `getDelegate` — whose own `receiveCommand` switch maps `"clearSelection"` + * and `"setSelection"` to the two methods below. An override that + * re-implemented that routing by hand would be a second copy of the wire + * format, which is the mistake the delegate exists to prevent; + * scripts/check-codegen.mjs pins the generated dispatcher so the route + * cannot vanish unnoticed. + */ + override fun clearSelection(view: SelectableRunHostView) { + view.clearSelection() + } + + override fun setSelection(view: SelectableRunHostView, start: Int, end: Int) { + // Straight through: clamping belongs in the view, which is the only + // place that knows what the text currently is. JS computed these + // offsets against a snapshot that can be a frame behind, exactly like + // the offsets travelling the other way on an event. + view.setSelection(start, end) + } + @ReactProp(name = "pressables") override fun setPressables(view: SelectableRunHostView, pressables: ReadableArray?) { // Parsed on arrival, like `attributes`: the ReadableArray is bridge @@ -151,10 +195,19 @@ class SelectableRunHostViewManager : view.setEmbeds(RunEmbeds.parse(embeds)) } + /** + * The ordered menu, one string per item: an action identifier, or + * `identifier + U+001F + title` (src/view/selectionActions.ts packs it, + * and the view unpacks it). The strings are copied out of the + * ReadableArray unexamined — splitting them here would put the wire + * format in two places, and the view has to know it anyway to resolve a + * title against its own string resources. + */ @ReactProp(name = "selectionActions") override fun setSelectionActions(view: SelectableRunHostView, actions: ReadableArray?) { if (actions == null) { - // Prop reset: fall back to the default menu (both actions). + // Prop reset: fall back to the default menu (both built-in + // actions, each taking the view's own localised title). view.setSelectionActions( listOf( SelectableRunHostView.ACTION_COPY_TEXT, @@ -204,9 +257,10 @@ class SelectableRunHostViewManager : * JNI call is in points, so the result has to be converted back. React * Native's own text measurement ends with the same * `PixelUtil.toDIPFromPixel` pair (TextLayoutManager.java:692-711). - * Skipping it would report every run three times too tall on a 3x device; - * the paper path takes `RunTextMeasure`'s pixels straight through, because - * paper's Yoga tree is in pixels to begin with. + * Skipping it would report every run three times too tall on a 3x device. + * `RunTextMeasure` itself stays in pixels and leaves the conversion here, + * because it is also what configures the `TextView`'s paint and the view + * side wants pixels. */ override fun measure( context: Context, @@ -219,6 +273,11 @@ class SelectableRunHostViewManager : heightMode: YogaMeasureMode, attachmentsPositions: FloatArray? ): Long { + // The earliest Context this component ever holds: Fabric measures + // before it mounts, so this — not createViewInstance — is what keeps + // a bundled `fontFamily` from being measured in the system fallback + // face. Idempotent and one volatile read after the first call. + RunTypefaces.install(context) val text = if (props != null && props.hasKey("text")) props.getString("text") ?: "" else "" val attributes = if (props != null && props.hasKey("attributes")) { RunAttributedText.parse(props.getArray("attributes")) @@ -294,6 +353,8 @@ class SelectableRunHostViewManager : mapOf("registrationName" to "onInlinePress") constants[EmbedLayoutEvent.EVENT_NAME] = mapOf("registrationName" to "onEmbedLayout") + constants[SelectionChangeEvent.EVENT_NAME] = + mapOf("registrationName" to "onSelectionChange") return constants } diff --git a/android/src/main/java/com/selectablemarkdown/SelectionChangeEvent.kt b/android/src/main/java/com/selectablemarkdown/SelectionChangeEvent.kt new file mode 100644 index 0000000..1ac0c33 --- /dev/null +++ b/android/src/main/java/com/selectablemarkdown/SelectionChangeEvent.kt @@ -0,0 +1,67 @@ +package com.selectablemarkdown + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.WritableMap +import com.facebook.react.uimanager.events.Event + +/** + * `onSelectionChange` — fired whenever the run's selection moves, including to + * nothing. + * + * Everything structural here is inherited from `SelectionActionEvent`'s + * reasoning, which is the authoritative copy: the `Event`-object dispatch + * (posted to the dispatcher `UIManagerHelper.getEventDispatcherForReactTag` + * resolves, so one path serves both architectures), and the `top…` spelling + * (matched verbatim on paper, left alone by Fabric's `normalizeEventType`, and + * what the codegen'd view config keys the event under). + * + * WHAT IS DIFFERENT FROM THE OTHER TWO EVENTS, and it is the whole reason this + * one exists: an EMPTY range is a legal payload. `SelectionActionEvent` never + * emits one — a menu item fired against no selection would be nonsense — but + * "the selection went away" is exactly what a consumer's own floating toolbar + * has to hear in order to dismiss itself, and nothing else reports it. + * + * `canCoalesce` is FALSE here too, and the reasoning is worth stating because + * this is the one event where coalescing looks defensible: it fires + * repeatedly while a selection handle is dragged, and keeping only the newest + * of two in a frame would be harmless for a drag. It is not harmless for the + * pair that matters — a hand-off emits "run B holds [4,9)" and then "run A + * holds nothing", two events from two different views. `Event` coalesces by + * name, view AND coalescing key (Event.java:100-112), so two views never + * collide and the pair would survive; what would not survive is a clear and a + * re-selection inside one view in a single frame, which is exactly the + * sequence a programmatic `setSelection` produces. The host's own dedupe is + * what keeps the volume down, and it drops only genuinely identical ranges. + * + * There is no `selectedText` field, unlike `SelectionActionEvent`. Building + * one would mean a substring per frame of a drag, transcoded across the + * bridge, for a string JS can already slice out of the text it sent. + */ +internal class SelectionChangeEvent( + surfaceId: Int, + viewId: Int, + private val start: Int, + private val end: Int, +) : Event(surfaceId, viewId) { + + override fun getEventName(): String = EVENT_NAME + + override fun canCoalesce(): Boolean = false + + /** + * `start`/`end` are UTF-16 offsets into the *current* `text`, + * end-exclusive, clamped and ordered by the emitter in + * `SelectableRunHostView` — the same contract as every other offset this + * component reports, except that `start == end` is meaningful here and + * means "nothing is selected". + */ + override fun getEventData(): WritableMap = + Arguments.createMap().apply { + putInt("start", start) + putInt("end", end) + } + + companion object { + const val EVENT_NAME = "topSelectionChange" + } +} diff --git a/android/src/main/jni/CMakeLists.txt b/android/src/main/jni/CMakeLists.txt index ae0edd1..ed9de64 100644 --- a/android/src/main/jni/CMakeLists.txt +++ b/android/src/main/jni/CMakeLists.txt @@ -205,13 +205,20 @@ target_include_directories(react_codegen_SelectableMarkdownSpec BEFORE PUBLIC # FabricUIManager.measure as a ReadableMap. # # PROBED BY TARGET EXISTENCE, NEVER BY VERSION NUMBER. React Native 0.73-0.75 -# publish a prefab module per feature (reactnativejni among them); 0.76 -# collapsed the lot into a single `reactnative`. Both names are aliased into -# the app's CMake scope by ReactNative-application.cmake before it includes -# Android-autolinking.cmake, so exactly one of these branches is live and it is -# the right one — with no version string in this file to go stale. This is the -# same discipline ../../../CMakeLists.txt applies to ReactAndroid::jsi, and the -# reason that file needs no version ladder either. +# published a prefab module per feature (reactnativejni among them); 0.76 +# collapsed the lot into a single `reactnative`, and that is the only branch +# reachable inside the peer range package.json declares today (>= 0.82). The +# `reactnativejni` branch is HISTORICAL — it is kept, not because a supported +# consumer can take it, but because it costs one line and is the only thing +# standing between a consumer who overrides the peer range and a page of +# undefined fbjni symbols. Whichever of the two names the app's React Native +# publishes is aliased into the app's CMake scope by +# ReactNative-application.cmake before it includes Android-autolinking.cmake +# (0.75.4 aliases `reactnativejni` there; 0.76+ aliases `reactnative`), so +# exactly one of these branches is live and it is the right one — with no +# version string this file has to keep current. This is the same discipline +# ../../../CMakeLists.txt applies to ReactAndroid::jsi, and the reason that +# file needs no version ladder either. # # Failing loudly rather than falling through: if neither target exists, the # link error would be a page of undefined facebook::jni symbols from a @@ -223,10 +230,11 @@ elseif(TARGET reactnative) target_link_libraries(react_codegen_SelectableMarkdownSpec reactnative) else() message(FATAL_ERROR - "react-native-selectable-markdown: neither the `reactnativejni` (React " - "Native 0.73-0.75) nor the `reactnative` (0.76+) prefab target is defined " - "in this scope. RNSMRunTextMeasurer.cpp needs react/jni/ReadableNativeMap.h " - "from one of them. If React Native renamed the module again, add the new " - "name as another branch here — the probe is deliberately by target " - "existence so this stays a one-line change.") + "react-native-selectable-markdown: neither the `reactnative` prefab target " + "(React Native 0.76+, which covers the whole supported range) nor the " + "historical `reactnativejni` (0.73-0.75) is defined in this scope. " + "RNSMRunTextMeasurer.cpp needs react/jni/ReadableNativeMap.h from one of " + "them. If React Native renamed the module again, add the new name as " + "another branch here — the probe is deliberately by target existence so " + "this stays a one-line change.") endif() diff --git a/android/src/main/jni/RNSMRunTextMeasurer.cpp b/android/src/main/jni/RNSMRunTextMeasurer.cpp index 0e224d6..da93a6c 100644 --- a/android/src/main/jni/RNSMRunTextMeasurer.cpp +++ b/android/src/main/jni/RNSMRunTextMeasurer.cpp @@ -18,9 +18,8 @@ * The consequence is the property docs/FABRIC-PLAN.md §4 cares most about: * measure/draw agreement on Android is not maintained by keeping two engines * configured identically, it is structural, because there is only one engine - * and both sides call it. Paper's `SelectableRunHostShadowNode` calls the same - * `RunTextMeasure.measure`, so a run does not even change height when an app - * flips `newArchEnabled`. + * and both sides call it — the shadow node through this file, and the mounted + * `SelectableRunHostView` directly. * * THIS FILE ENCODES NOTHING AND KNOWS NOTHING ABOUT ATTRIBUTES. It forwards * `props.rawProps` — the exact `folly::dynamic` of the JS props, populated by diff --git a/android/src/main/res/values/strings.xml b/android/src/main/res/values/strings.xml new file mode 100644 index 0000000..158894d --- /dev/null +++ b/android/src/main/res/values/strings.xml @@ -0,0 +1,31 @@ + + + + Copy Text + Copy Markdown + diff --git a/bench/crossing.mjs b/bench/crossing.mjs index 85bbf78..dd9e6b0 100644 --- a/bench/crossing.mjs +++ b/bench/crossing.mjs @@ -8,7 +8,10 @@ // allocation of the buffer that crosses. // (b) JS decode -> AST — decodeFlatBuffer over a PRE-COMPUTED buffer, so // nothing native is running inside the measurement. This is the JS half -// of the protocol: span widening, text slicing, entity decoding, policy. +// of the protocol: span widening, text slicing, the string table's UTF-8 +// decode, policy. Entity resolution is NOT here — md4c resolves entities +// and the native side interns the decoded value (OffsetParser.cpp), so +// the JS side only reads it out of the string table. // (c) total — the engine's real parse() call, (a) + (b) plus the glue. // // Reading it: if (a) dominates, the parser is the bottleneck and a faster @@ -53,7 +56,11 @@ const P = loadNativeProtocol(); const { decodeFlatBuffer, resolveOptions, presets } = lib; const options = resolveOptions(presets.llmChat); const extBits = P.extensionBits(options); -const htmlPolicy = P.htmlPolicyBit(options); +// No argument: `htmlPolicyBit()` takes none, on purpose. The native side +// ignores the word and `html` is applied in the decoder, so nothing about the +// resolved options can change it — passing `options` here read as if the +// mapping were still open. +const htmlPolicy = P.htmlPolicyBit(); const rawParse = native.parse; const engine = native.engine; diff --git a/bench/gates.test.ts b/bench/gates.test.ts new file mode 100644 index 0000000..a9feb11 --- /dev/null +++ b/bench/gates.test.ts @@ -0,0 +1,336 @@ +/** + * The bench harness as a GATE, not as a report. + * + * Three of these benches are wired into .github/workflows/ci.yml, where their + * exit codes decide whether a pull request is red. That makes their failure + * paths product behaviour: a gate that exits 0 on an empty run, or dies with a + * TypeError on a flag combination, is worse than no gate at all — it is a + * green check over nothing. So the cases below are the ones nobody exercises + * by hand: no samples, no engine, a budget that must trip, a growth limit that + * must trip. + * + * Everything is a subprocess. The benches are ESM scripts with top-level + * `await` and `process.exit`, and the exit code IS the thing under test. + */ +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const repoRoot = path.resolve(__dirname, '..'); +const benchDir = path.join(repoRoot, 'bench'); +const sprintTranscript = path.join( + repoRoot, + 'conformance', + 'fixtures', + 'transcript-sprint-review.json', +); +// The never-anchoring shape: one 420-item bullet list, which no blank line +// closes, so every append re-reads the whole accumulated text. It is the +// transcript the workflows gate. +const giantTranscript = path.join( + repoRoot, + 'conformance', + 'fixtures', + 'transcript-giant-list.json', +); +const ciWorkflow = path.join(repoRoot, '.github', 'workflows', 'ci.yml'); +const releaseWorkflow = path.join(repoRoot, '.github', 'workflows', 'release.yml'); + +interface Ran { + status: number | null; + stdout: string; + stderr: string; + output: string; +} + +const runNode = (args: readonly string[], env?: NodeJS.ProcessEnv): Ran => { + const result = spawnSync(process.execPath, [...args], { + cwd: repoRoot, + encoding: 'utf8', + env: { ...process.env, ...env }, + }); + const stdout = result.stdout ?? ''; + const stderr = result.stderr ?? ''; + return { status: result.status, stdout, stderr, output: `${stdout}${stderr}` }; +}; + +const pathological = path.join(benchDir, 'pathological.mjs'); +const projection = path.join(benchDir, 'projection.mjs'); +const streaming = path.join(benchDir, 'streaming-replay.mjs'); + +// The benches need dist/ and the compiled addon. Both exist in CI (npm ci runs +// `prepare`, and the addon build is a hard gate before `npm test`) and on any +// machine that has run `npm run build`. Where they do not, `bench/support.mjs` +// would spawn a full `npm run build` inside a test — so the suite reports the +// missing prerequisite instead of timing out on it. +const distReady = fs.existsSync(path.join(repoRoot, 'dist', 'engine', 'Engine.js')); +const addonReady = fs.existsSync( + path.join(repoRoot, 'build', `selectable-markdown.${process.platform}-${process.arch}.node`), +); +const describeBench = distReady && addonReady ? describe : describe.skip; + +describeBench('bench:pathological as a gate', () => { + it('reports rather than crashing when --runs asks for no samples', () => { + const ran = runNode([pathological, '--quick', '--runs', '0']); + + // The regression: the no-samples branch dereferenced `crash.stage` with no + // null check, so this exact command died with + // `TypeError: Cannot read properties of null (reading 'stage')`. + expect(ran.stderr).not.toContain('TypeError'); + expect(ran.stderr).not.toContain("reading 'stage'"); + expect(ran.status).toBe(0); + expect(ran.stdout).toContain('no samples'); + }); + + it('fails a --budget run that produced no samples, instead of passing over nothing', () => { + const ran = runNode([pathological, '--quick', '--runs', '0', '--budget', '1000']); + + expect(ran.status).toBe(1); + expect(ran.stdout).toContain('a gate over nothing is a failure'); + }); + + it('gates each stage at its own budget', () => { + // `segment` is tens of microseconds on these inputs, so under one global + // budget loose enough for the parse it could get a hundred times slower + // and still pass. A per-stage budget is the only thing that sees it. + const ran = runNode([pathological, '--quick', '--budget-segment', '0.0001']); + + expect(ran.status).toBe(1); + expect(ran.stdout).toMatch(/OVER\s+segment/); + expect(ran.stdout).toContain('budget exceeded'); + + // The other stages are ungated in that run: passing one override must not + // silently apply it everywhere. + expect(ran.stdout).toMatch(/ok\s+parse/); + }); + + it('passes the workflow invocation on this machine', () => { + // Exactly what .github/workflows/ci.yml runs, so a budget tightened below + // what the library actually costs turns this suite red before it turns + // every pull request red. + const ran = runNode([ + pathological, + '--require-engine', + '--budget-parse', + '750', + '--budget-repair', + '750', + '--budget-segment', + '200', + '--budget-project', + '750', + ]); + + expect(ran.output).not.toContain('measured nothing'); + expect(ran.status).toBe(0); + }); +}); + +describe('--require-engine', () => { + // The failure path cannot be reached by breaking the addon (the harness + // rebuilds it), so it is exercised where it is implemented: the helper every + // gated bench calls when its engine came back null. + const probe = (flags: readonly string[]): Ran => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'rnsm-require-engine-')); + const file = path.join(dir, 'probe.mjs'); + const support = new URL(`file://${path.join(benchDir, 'support.mjs')}`).href; + fs.writeFileSync( + file, + `import { exitWithoutEngine } from ${JSON.stringify(support)};\n` + + "exitWithoutEngine('[probe]');\n", + ); + try { + return runNode([file, ...flags]); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }; + + it('exits 0 without the flag — a laptop with no toolchain is not a regression', () => { + const ran = probe([]); + expect(ran.status).toBe(0); + expect(ran.stdout).toContain('pass --require-engine'); + }); + + it('exits 1 with the flag, so a CI gate cannot pass having measured nothing', () => { + const ran = probe(['--require-engine']); + expect(ran.status).toBe(1); + expect(ran.stderr).toContain('measured nothing'); + }); +}); + +describeBench('bench:projection as a gate', () => { + it('passes on the anchoring transcript and prints the threshold it gated on', () => { + const ran = runNode([projection, '--transcript', sprintTranscript, '--require-engine']); + + expect(ran.status).toBe(0); + expect(ran.stdout).toMatch(/gate: cached <= 1\.25x/); + expect(ran.stdout).toMatch(/growth {2}cached=/); + }); + + it('exits 1 when cached growth passes the limit', () => { + // The gate fires on a number this corpus does not produce (cached growth + // measures ~1.00), so the limit is lowered in a copy of the bench rather + // than the corpus being bent to produce a regression. The copy lives in + // bench/ because it imports ./support.mjs relatively. + const copy = path.join(benchDir, `.growth-gate-probe.${process.pid}.mjs`); + const source = fs.readFileSync(projection, 'utf8').replace( + 'const GROWTH_LIMIT = 1.25;', + 'const GROWTH_LIMIT = 0.5;', + ); + expect(source).toContain('const GROWTH_LIMIT = 0.5;'); + fs.writeFileSync(copy, source); + try { + const ran = runNode([copy, '--transcript', sprintTranscript]); + expect(ran.status).toBe(1); + expect(ran.stdout).toMatch(/growth {2}OVER/); + expect(ran.stderr).toContain('cached amplification grew'); + } finally { + fs.rmSync(copy, { force: true }); + } + }); + + it('has no gate to run under --quick, and says so rather than passing quietly', () => { + // One document size means no growth ratio: the run must not print a gate + // verdict it did not compute. + const ran = runNode([projection, '--quick', '--transcript', sprintTranscript]); + expect(ran.status).toBe(0); + expect(ran.stdout).not.toMatch(/growth {2}cached=/); + }); +}); + +describeBench('bench:streaming reports matched statistics', () => { + // The ratio used to divide a SUM of every append time by a MEDIAN full parse + // times the chunk count — a statistic mismatch that made the printed figure + // move with this machine's noise, and let two docs quote it in opposite + // directions. Both sides are now the median of per-replay totals. + const ran = (): Ran => runNode([streaming, '--quick', '--transcript', sprintTranscript]); + + it('prints both sides of the ratio as the same statistic over the same count', () => { + const out = ran().stdout; + + const streamed = out.match( + /streamed\s+([\d.]+) ms = median of (\d+) replay\(s\), each the SUM of its (\d+) append times/, + ); + const naive = out.match( + /naive\s+([\d.]+) ms = median of (\d+) replay\(s\), each the SUM of (\d+) full reparses/, + ); + expect(streamed).not.toBeNull(); + expect(naive).not.toBeNull(); + + // Same estimator (median of per-replay sums), same number of replays, same + // chunk count on both sides. Any of the three differing is the defect. + expect(naive![2]).toBe(streamed![2]); + expect(naive![3]).toBe(streamed![3]); + }); + + it('prints a ratio that is the quotient of the two numbers beside it', () => { + const out = ran().stdout; + const ratio = Number(out.match(/incremental-vs-full reparse ratio: ([\d.]+)/)![1]); + const streamedMs = Number(out.match(/streamed\s+([\d.]+) ms = median/)![1]); + const naiveMs = Number(out.match(/naive\s+([\d.]+) ms = median/)![1]); + + // Both totals are printed to two decimals, so the quotient can only be + // recomputed to within their rounding — compared relatively for that + // reason. What this pins is that the ratio is those two terms and not some + // third quantity the reader cannot see (the defect it replaces was off by + // the ratio of a sum to a median, which is not a few percent). + const recomputed = streamedMs / naiveMs; + expect(Math.abs(ratio - recomputed) / recomputed).toBeLessThan(0.1); + }); +}); + +describeBench('bench:streaming as a gate', () => { + // Until --budget existed this bench had no threshold, no non-zero exit and no + // workflow: the adversarial STREAMING shape was reported and never gated, + // while bench/pathological.mjs's header claimed `bench:streaming` owned the + // question. These are the paths that make the claim true. + const giant = (flags: readonly string[]): Ran => + runNode([streaming, '--transcript', giantTranscript, '--repeat', '1', ...flags]); + + it('exits 1 when a chunk p99 passes its budget', () => { + // The budget is set below what any machine can hit rather than the corpus + // being bent into a regression. + const ran = giant(['--budget-chunk', '0.0001']); + + expect(ran.status).toBe(1); + expect(ran.stdout).toMatch(/OVER\s+chunk p99/); + expect(ran.stdout).toContain('budget exceeded'); + }, 120_000); + + it('leaves the number nobody budgeted ungated', () => { + // Passing one override must not silently gate everything, the same rule + // bench:pathological's per-stage budgets follow. + const ran = giant(['--budget-chunk', '0.0001']); + expect(ran.stdout).toMatch(/off\s+finalize/); + }, 120_000); + + it('fails a budgeted run that timed nothing, instead of passing over zero samples', () => { + const ran = giant(['--max-chunks', '0', '--budget-chunk', '20']); + + expect(ran.status).toBe(1); + expect(ran.stdout).toContain('a gate over nothing is a failure'); + }, 120_000); + + it('reports rather than failing when no budget is passed', () => { + // `npm run bench:streaming` on a laptop is a report: the absolute + // milliseconds belong to the machine. + const ran = giant([]); + expect(ran.status).toBe(0); + expect(ran.stdout).not.toContain('budget exceeded'); + }, 120_000); + + it('passes the workflow invocation on this machine', () => { + // Exactly what both workflows run, so a budget tightened below what the + // library actually costs turns this suite red before it turns every pull + // request red. + const ran = giant(['--require-engine', '--budget-chunk', '20', '--budget-finalize', '50']); + + expect(ran.output).not.toContain('measured nothing'); + expect(ran.stdout).toMatch(/ok\s+chunk p99/); + expect(ran.stdout).toMatch(/ok\s+finalize/); + expect(ran.status).toBe(0); + }, 120_000); +}); + +describe('bench:streaming is wired in as a gate, not only implemented as one', () => { + const source = fs.readFileSync(streaming, 'utf8'); + + it('exits through exitWithoutEngine rather than a bare process.exit(0)', () => { + // A step that exits 0 having measured nothing is a green gate over zero + // samples — the exact hole --require-engine closed for the other benches. + // This file kept two bare `process.exit(0)` calls (unresolvable engine, + // unusable StreamSession), so wiring it into CI as it stood would have + // reintroduced it. + expect(source).not.toContain('process.exit(0)'); + expect(source).toContain('exitWithoutEngine'); + }); + + it('runs in ci.yml and release.yml with a budget and --require-engine', () => { + for (const file of [ciWorkflow, releaseWorkflow]) { + const runnable = fs + .readFileSync(file, 'utf8') + .split('\n') + .filter((line) => !line.trim().startsWith('#')) + .join('\n'); + + expect(runnable).toContain('npm run bench:streaming'); + const step = runnable.slice(runnable.indexOf('npm run bench:streaming')); + const flags = step.slice(0, step.indexOf('- run:')); + expect(flags).toContain('conformance/fixtures/transcript-giant-list.json'); + expect(flags).toContain('--require-engine'); + expect(flags).toMatch(/--budget-chunk \d/); + expect(flags).toMatch(/--budget-finalize \d/); + } + }); + + it("no longer claims bench:streaming gates nothing", () => { + // bench/pathological.mjs justifies having no streaming stage by pointing at + // this bench. That sentence was true about the harness and false about the + // gate until the budget existed. + const pathologicalSource = fs.readFileSync(pathological, 'utf8'); + expect(pathologicalSource).toContain('`bench:streaming` owns that'); + expect(pathologicalSource).toMatch(/gates it/); + }); +}); diff --git a/bench/head-to-head.mjs b/bench/head-to-head.mjs index 8099f85..a3c824b 100644 --- a/bench/head-to-head.mjs +++ b/bench/head-to-head.mjs @@ -46,10 +46,22 @@ import { loadLibrary, loadNativeEngine, numberFlag, + refuseEngineFlag, repoRoot, stats, } from './support.mjs'; +// Before anything else, and before the `--only` child dispatch below, because +// this bench is the one whose output is a LEADERBOARD: printing md4c's row +// under a heading a stale `--engine reference` made the reader expect is the +// exact misreading refuseEngineFlag exists to prevent. The other four benches +// get this through resolveEngine; head-to-head resolves the engine itself +// (loadNativeEngine directly, so an unbuildable addon is one skipped row +// rather than the end of the run), which is how it went unrefused. Children +// are spawned with an explicitly built argv rather than an inherited one, so +// refusing here refuses for the whole run. +refuseEngineFlag(); + const replicas = numberFlag('replicas', 12); const iterations = numberFlag('iterations', hasFlag('quick') ? 3 : 20); const warmup = hasFlag('quick') ? 1 : 3; diff --git a/bench/pathological.mjs b/bench/pathological.mjs index 715b10a..204cd55 100644 --- a/bench/pathological.mjs +++ b/bench/pathological.mjs @@ -1,17 +1,79 @@ #!/usr/bin/env node // Pathological-input timing: adversarial documents that punish quadratic -// parsers. Reports per-case wall-clock; with --budget it becomes a DoS -// regression gate (exit 1 when any case exceeds the budget or crashes). +// parsers. Reports per-case, per-STAGE wall-clock; with --budget it becomes a +// DoS regression gate (exit 1 when any stage exceeds the budget or crashes). // -// Usage: node bench/pathological.mjs [--quick] [--budget MS] [--runs N] +// Usage: node bench/pathological.mjs [--quick] [--runs N] [--require-engine] +// [--budget MS] [--budget-parse MS] [--budget-repair MS] +// [--budget-segment MS] [--budget-project MS] // -// --budget turns the run into a gate: any case whose median exceeds it, or +// --budget turns the run into a gate: any stage whose median exceeds it, or // that throws, exits 1. Without it the cases are only reported, because the // absolute numbers depend on the machine and a bare `npm run bench:*` should // not fail on a busy laptop. +// +// ONE BUDGET IS THE WRONG SHAPE FOR FOUR STAGES that differ by four orders of +// magnitude. `segment` is tens of microseconds on these inputs and `parse` is +// tens of milliseconds, so a single number loose enough for the parse (CI ran +// 1000 ms) is ~20000x the segment's real cost: that stage could get a hundred +// times slower and still pass. `--budget-` overrides the global for one +// stage, so each is gated near its own scale and the workflows pass four +// numbers instead of one. The global remains the default for any stage with no +// override, and passing only overrides gates only those stages. +// +// --require-engine turns "the addon did not resolve" from an exit-0 report +// into a failure. A gate that exits 0 having measured nothing is worse than no +// gate, and the likeliest cause here is not a missing compiler but a +// protocol-version drift between the built addon and dist/. Workflows pass it; +// a laptop with no toolchain should not. +// +// WHY THIS TIMES FOUR STAGES AND NOT JUST THE PARSE +// ------------------------------------------------- +// md4c is linear on every shape below — that is the whole reason it was +// picked, and it means a parse-only gate is a gate on the one stage that was +// never going to fail. Everything a consumer runs *after* the parse is +// TypeScript over the decoded tree, and none of it is obviously linear: +// +// parse md4c + the FlatBuffer decode. The linear one. +// repair `repairTail` over the whole input, which is what a stream that +// never anchors actually hands it (a list, a giant paragraph, an +// unclosed fence — see docs/BENCHMARKS.md). Its scanners walk the +// tail per construct, so this is where an adversarial run of +// emphasis openers costs far more than parsing them. +// segment `segmentRuns` over every block: the per-snapshot cost the view +// pays before anything renders. +// project `projectRun` over every run: the per-run cost that produces the +// text the native hosts actually measure and select. It walks the +// block tree, so deep nesting is priced here rather than in the +// parse. +// +// A throw anywhere in the four is a CRASH for that case, and --budget fails on +// it: a `RangeError: Maximum call stack size exceeded` on 3 kB of `> ` is a +// denial of service whatever its runtime, and reporting it as a fast case +// would be the worst possible reading of these numbers. +// +// Streaming is deliberately NOT a stage here. `bench:streaming` owns that +// question and gates it: it replays the pinned never-anchoring transcript +// (conformance/fixtures/transcript-giant-list.json) — the adversarial +// *streaming* shape, in the same way these four are the adversarial *document* +// shapes — and takes the same `--budget`/`--require-engine` flags, so ci.yml +// and release.yml run it in the step next to this one. Until it did, the +// sentence here pointed at a report and called it coverage. +// +// AND WHY THERE IS A SECOND, SCALING SECTION +// ------------------------------------------ +// The four cases above are each ONE size, so they price a shape but cannot +// tell a slow linear pass from a fast quadratic one — and a quadratic pass is +// exactly what the repair's unmatched-bracket strip loop used to be. The +// `repair scaling` section below therefore runs one shape at five sizes and +// reports each against a linear control of the same length, because the ratio +// between them is machine-independent in a way wall-clock milliseconds are +// not: a flat ratio column is linear, a doubling one is quadratic, and that +// reading holds on a busy laptop and in CI alike. import { deepBlockquoteSource, + exitWithoutEngine, fmtBytes, fmtMs, hasFlag, @@ -27,13 +89,19 @@ const budgetMs = numberFlag('budget', undefined); const runs = numberFlag('runs', quick ? 1 : 3); const lib = loadLibrary(); -const { parseDocument, presets } = lib; +const { parseDocument, presets, projectRun, repairTail, resolveOptions, segmentRuns } = lib; const engine = await resolveEngine(lib, '[bench:pathological]'); -if (!engine) process.exit(0); +if (!engine) exitWithoutEngine('[bench:pathological]'); const scale = (full, small) => (quick ? small : full); +// The seed `repairTail` gets from a StreamSession whose anchor sits at a clean +// boundary — no open fence, not inside math. Identical to `CLEAN_SEED` in +// src/stream/StreamSession.ts, and the only seed reachable there, because an +// anchor is accepted only with a clean fence/math scan state. +const CLEAN_SEED = { openFence: null, inMath: false }; + const cases = [ { name: 'nested brackets', @@ -57,53 +125,265 @@ const cases = [ }, ]; +/** + * The pipeline, split at the seams a consumer actually crosses. + * + * Each stage is timed on its own so a regression names the stage it is in; + * timing the four together would only say "slower". `run` receives the + * carry-over from the stages before it (the parsed document, then the runs), + * because re-parsing per stage would price the parse four times. + */ +const STAGES = [ + { + name: 'parse', + run: (c, state) => { + state.doc = parseDocument(c.input, c.options, engine.engine); + }, + }, + { + name: 'repair', + run: (c, state) => { + repairTail(c.input, CLEAN_SEED, state.resolved, undefined); + }, + }, + { + name: 'segment', + run: (c, state) => { + state.runs = segmentRuns(state.doc); + }, + }, + { + name: 'project', + run: (c, state) => { + for (const segment of state.runs) projectRun(segment, state.doc); + }, + }, +]; + +/** + * The budget each stage is gated at: `--budget-` when given, otherwise + * the global `--budget`, otherwise none (report only). + * + * Resolved once, up front, so the header line can print exactly what is being + * gated — a gate whose thresholds are only visible by reading the source is + * one nobody re-tunes when the numbers move. + */ +const budgets = new Map( + STAGES.map((stage) => [stage.name, numberFlag(`budget-${stage.name}`, budgetMs)]), +); +const gating = [...budgets.values()].some((ms) => ms !== undefined); + let anyOver = false; let anyCrash = false; +// A gate that produced no samples at all (`--runs 0`) is vacuous, which is the +// same failure as `--require-engine` catching an unresolvable addon: green, +// and over nothing. Tracked separately from `anyOver` so the message can say +// which of the two happened. +let anyVacuous = false; -console.log( - `pathological inputs (${runs} run(s) per case${quick ? ', quick' : ''}${ - budgetMs !== undefined ? `, budget ${budgetMs} ms` : '' - })`, -); +const budgetSummary = gating + ? `, budgets ${STAGES.map((stage) => { + const ms = budgets.get(stage.name); + return `${stage.name} ${ms === undefined ? 'off' : `${ms} ms`}`; + }).join(', ')}` + : ''; + +console.log(`pathological inputs (${runs} run(s) per case${quick ? ', quick' : ''}${budgetSummary})`); for (const c of cases) { - const times = []; + console.log(` ${c.name.padEnd(30)} ${fmtBytes(Buffer.byteLength(c.input, 'utf8')).padStart(9)}`); + + const times = new Map(STAGES.map((s) => [s.name, []])); let crash = null; // Deliberately no warmup: these inputs are about worst-case cold behaviour, // and a warmed-up JIT is not what a DoS attempt meets. The first throw ends // the case — the remaining runs would only reproduce it, and the timings - // collected before it are not comparable to a case that completed. + // collected before it are not comparable to a case that completed. The + // stage it threw in is kept, because "which stage" is most of the answer. for (let i = 0; i < runs && !crash; i += 1) { - const t0 = performance.now(); - try { - parseDocument(c.input, c.options, engine.engine); - times.push(performance.now() - t0); - } catch (err) { - crash = err; + const state = { resolved: resolveOptions(c.options), doc: null, runs: [] }; + for (const stage of STAGES) { + const t0 = performance.now(); + try { + stage.run(c, state); + } catch (err) { + crash = { stage: stage.name, err }; + break; + } + times.get(stage.name).push(performance.now() - t0); } } - console.log(` ${c.name.padEnd(30)} ${fmtBytes(Buffer.byteLength(c.input, 'utf8')).padStart(9)}`); - - let status; - let detail; - if (crash) { - anyCrash = true; - status = 'CRASH'; - detail = String(crash.message || crash).split('\n')[0]; - } else { - const median = percentile(times, 50); - const over = budgetMs !== undefined && median > budgetMs; + for (const stage of STAGES) { + const samples = times.get(stage.name); + // A stage that threw is a CRASH even when an earlier run of it completed: + // one input, one throw, and averaging that away is how a gate stops + // gating. + if (crash && crash.stage === stage.name) { + anyCrash = true; + console.log( + ` ${'CRASH'.padEnd(5)} ${stage.name.padEnd(8)} ${String(crash.err.message || crash.err).split('\n')[0]}`, + ); + continue; + } + if (samples.length === 0) { + // Two ways to get here, and `crash` is null in one of them: a stage + // after the throw was never reached, or `--runs 0` asked for no runs at + // all. Dereferencing `crash.stage` unconditionally is what made + // `--runs 0` die with a TypeError instead of reporting. + if (crash) { + console.log(` ${'n/a'.padEnd(5)} ${stage.name.padEnd(8)} not reached — \`${crash.stage}\` threw`); + } else { + anyVacuous = true; + console.log( + ` ${'none'.padEnd(5)} ${stage.name.padEnd(8)} no samples — \`--runs ${runs}\` asked for none`, + ); + } + continue; + } + const median = percentile(samples, 50); + const budget = budgets.get(stage.name); + const over = budget !== undefined && median > budget; if (over) anyOver = true; - status = over ? 'OVER' : 'ok'; - detail = `median ${fmtMs(median)} (min ${fmtMs(Math.min(...times))}, max ${fmtMs(Math.max(...times))})`; + console.log( + ` ${(over ? 'OVER' : 'ok').padEnd(5)} ${stage.name.padEnd(8)} median ${fmtMs(median)} ` + + `(min ${fmtMs(Math.min(...samples))}, max ${fmtMs(Math.max(...samples))})` + + `${budget === undefined ? '' : ` vs ${budget} ms`}`, + ); + } +} + +// --------------------------------------------------------------------------- +// repair scaling: unmatched brackets against a linear control +// --------------------------------------------------------------------------- + +/* + * `'x [ '` repeated is the shape that used to make `repairTail` quadratic: every + * `[` opens a link candidate that never closes, so the repair ends up with a + * stack of openers to strip, and stripping them one at a time — each strip + * rescanning the rest of the tail — is O(brackets * tail). Nothing above would + * have caught it. `nested brackets` is 10 000 `[` in a row followed by 10 000 + * `]`, which is a different (matched, deeply nested) shape, and it is measured + * at one size, so a quadratic pass there just reads as "repair is slow on + * brackets". + * + * `'x y '` is the control: identical length, identical word/space rhythm, no + * construct characters at all. Dividing by it cancels the per-character cost of + * simply walking the tail, so what is left is the price of the brackets — and + * that price must not grow with n. + * + * HOW TO READ IT. The `xctl` column is how much the brackets cost over the bare + * walk — tens of times the control, and roughly FLAT across the five sizes when + * the pass is linear, climbing with each doubling of n when it is not. It is + * coarse, because the control is microseconds and the clock is not much finer, + * so the gated number below it divides the bracket cost by n instead. The + * milliseconds are for scale only: ~1-2 ms at n = 8000 after the fix, ~17 ms + * before it. + */ + +const REPAIR_SCALE_NS = quick ? [500, 1_000, 2_000] : [500, 1_000, 2_000, 4_000, 8_000]; +const REPAIR_SCALE_OPTIONS = resolveOptions(presets.commonmark); + +/** + * Median cost of one `repairTail` over the whole tail, in ms. + * + * THE WARMUP IS THE ONE PLACE THIS FILE WANTS ONE, and it is not a + * contradiction of the no-warmup rule above. The four cases up there each + * report an absolute cost at one size, where cold is the honest number. This + * section reports a SHAPE across five sizes, and a cold first sample lands + * entirely on the smallest n — the one every later size is compared against — + * so an unwarmed run reads as the smallest input being the slowest and says + * nothing at all about growth. + * + * `batch` repeats the call inside the timed region and divides, which is only + * for the control: at these sizes one pass over 2 kB of `'x y '` costs about as + * much as `performance.now()` can resolve, and a quantised denominator makes + * the ratio column wobble by 2x for no reason. The bracket side is milliseconds + * on its own and is timed one call at a time. + */ +const REPAIR_SCALE_SAMPLES = Math.max(runs, 5); + +function medianRepairMs(input, batch = 1) { + repairTail(input, CLEAN_SEED, REPAIR_SCALE_OPTIONS, undefined); + const samples = []; + for (let i = 0; i < REPAIR_SCALE_SAMPLES; i += 1) { + const t0 = performance.now(); + for (let j = 0; j < batch; j += 1) { + repairTail(input, CLEAN_SEED, REPAIR_SCALE_OPTIONS, undefined); + } + samples.push((performance.now() - t0) / batch); } - console.log(` ${status.padEnd(5)} ${engine.name.padEnd(10)} ${detail}`); + return percentile(samples, 50); +} + +/** Repeats of the control per timed region — see `medianRepairMs`. */ +const CONTROL_BATCH = 32; + +console.log(''); +console.log( + `repair scaling: unmatched brackets vs. a linear control ` + + `(${REPAIR_SCALE_SAMPLES} run(s) per size, after a warmup)`, +); + +const perBracket = []; +for (const n of REPAIR_SCALE_NS) { + const brackets = 'x [ '.repeat(n); + const control = 'x y '.repeat(n); + const bracketMs = medianRepairMs(brackets); + const controlMs = medianRepairMs(control, CONTROL_BATCH); + // A control fast enough to round to zero would make the ratio meaningless + // rather than large; report it as unavailable instead of dividing by it. + const ratio = controlMs > 0 ? bracketMs / controlMs : null; + perBracket.push({ n, msPerBracket: bracketMs / n }); + // The repair stage's budget, not the global one: this section times + // `repairTail` and nothing else. + const repairBudget = budgets.get('repair'); + const over = repairBudget !== undefined && bracketMs > repairBudget; + if (over) anyOver = true; + console.log( + ` ${(over ? 'OVER' : 'ok').padEnd(5)} n=${String(n).padStart(5)} ` + + `${fmtBytes(Buffer.byteLength(brackets, 'utf8')).padStart(9)} ` + + // The control is microseconds at these sizes; `fmtMs` would print + // every row as `0.00 ms`. + `brackets ${fmtMs(bracketMs)} control ${(controlMs * 1000).toFixed(0).padStart(4)} us ` + + `xctl ${ratio === null ? ' n/a' : ratio.toFixed(1).padStart(6)}`, + ); +} + +// The linearity read, made explicit so nobody has to eyeball the column. +// +// It divides the BRACKET cost by n rather than dividing the ratio column by +// itself, and the difference matters: the control is the smallest quantity on +// the line, so a drift computed from `xctl` inherits all of the control's +// timer noise on top of the signal. Cost per bracket is flat when the pass is +// linear (a constant amount of work per `[`) and grows with n when it is not, +// and it is read off the one column that is comfortably above the clock's +// resolution. +// +// 2.0 is the threshold, against a 16x growth in input from the first size to +// the last: generous enough that JIT warmth and a busy laptop cannot trip it, +// far below the ~16x a restored quadratic loop would show. +const GROWTH_LIMIT = 2; +if (perBracket.length >= 2) { + const first = perBracket[0]; + const last = perBracket[perBracket.length - 1]; + const growth = last.msPerBracket / first.msPerBracket; + const grew = growth > GROWTH_LIMIT; + if (grew) anyOver = true; + console.log( + ` ${(grew ? 'OVER' : 'ok').padEnd(5)} cost per bracket ${growth.toFixed(2)}x ` + + `from n=${first.n} to n=${last.n} (flat ~1.00x is linear; ` + + `${(last.n / first.n).toFixed(0)}x would be quadratic)`, + ); } -if (budgetMs !== undefined && (anyOver || anyCrash)) { - console.log('budget exceeded — failing.'); +if (gating && (anyOver || anyCrash || anyVacuous)) { + console.log( + anyVacuous && !anyOver && !anyCrash + ? `no stage produced a sample (--runs ${runs}) — a gate over nothing is a failure.` + : 'budget exceeded — failing.', + ); process.exit(1); } if (anyCrash) { diff --git a/bench/projection.mjs b/bench/projection.mjs new file mode 100644 index 0000000..220007e --- /dev/null +++ b/bench/projection.mjs @@ -0,0 +1,302 @@ +#!/usr/bin/env node +// Projection amplification: how many source characters the view layer hands +// `projectRun` over a whole streamed message, against how many characters the +// message contains. +// +// WHY THIS NUMBER, AND WHY NO OTHER BENCH SEES IT +// ---------------------------------------------- +// bench/throughput.mjs times a parse, bench/streaming-replay.mjs times an +// append, bench/crossing.mjs times the JS<->native hop, bench/pathological.mjs +// times one segment+project pass over a finished document. Not one of them +// replays the VIEW: segment the snapshot, then project each run, on every +// commit, the way `SelectableMarkdown` does. That is where the library's one +// superlinear step lived. +// +// The shape of it: `segmentRuns` merges every adjacent settled flowing block +// into one run, so an ordinary answer is ONE run that gains a block each time +// the stream settles. Reprojecting the whole run per settle costs O(document) +// per settle and O(document^2) over the message — the audit measured 47.7x the +// document at 14 kB and 90.1x at 28 kB, with a single late settle reprojecting +// 25 kB. `projectRun`'s `previous` option and `createRunProjectionCache` make +// growth cost the growth instead. +// +// So this bench prints, for each transcript and at two document sizes: +// +// projected/doc total source characters projected / document length. The +// invariant is that this ratio is FLAT in document size: it +// is set by how long a block spends as the redrawn tail +// (a function of delta size), not by the document in front +// of it. +// worst the largest single projection. Bounded by the tail plus +// the block that just settled — never the whole message. +// growth the ratio between the two sizes' amplification. ~1.0 is +// linear; the old design roughly doubled it per doubling. +// +// Both pipelines are measured side by side — `cached` is what ships, `full` is +// the same replay with the cache taken away — because the number only means +// something next to the one it replaced. +// +// TWO TRANSCRIPTS, AND THE SECOND ONE IS THE HONEST HALF. Incremental +// projection can only help a run whose blocks SETTLE, because a settled block +// is the same object on the next tick and that identity is the whole reuse +// test. `transcript-giant-list.json` is one 420-item bullet list, and a list +// never anchors (`StreamSession.isAnchorSafe`), so the entire document is one +// unsettled tail block that is reparsed — new object, new spans — on every +// delta. Nothing here can reuse anything, and its amplification stays enormous +// on both pipelines. That is the same pathology bench/streaming-replay.mjs +// exists to keep visible, one layer up; quoting only the first transcript's +// numbers as a property of the library is the mistake both benches refuse to +// let anyone make. +// +// `ms` is one un-warmed pass over the whole replay, printed for scale only. +// The counts are the measurement; they are exact and deterministic. +// +// THIS IS A GATE, NOT A REPORT, and it can be one precisely because the counts +// are exact. Nothing here is a wall-clock threshold that a busy runner can +// trip: `projected/doc` is a count of source characters handed to the +// projector, so the same transcript at the same `--chunk` gives the same +// number on every machine. So `cached` growth above GROWTH_LIMIT (below) exits +// 1 — the epilogue used to call itself "the gate" while never returning +// anything but 0, and no workflow ran it at all. +// +// Usage: node bench/projection.mjs [--quick] [--transcript PATH] [--chunk N] +// [--require-engine] +// +// --quick measures one document size, which leaves no growth ratio to compare +// and therefore nothing to gate — the gate needs both sizes. --require-engine +// turns a missing addon (or a dist/ built before src/view/projectionCache.ts) +// from an exit-0 report into a failure, so a CI step cannot pass having +// measured nothing. +// +// The counting instrument is an `EmbedLookup` that claims nothing: the +// projector offers a run's own blocks with `topLevel: true`, so summing their +// spans is exactly "source characters projected", and claiming nothing leaves +// the projection byte-identical (conformance/selection/incremental-projection.test.ts +// asserts that equivalence over the whole corpus). + +import { existsSync, readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import path from 'node:path'; + +import { + exitWithoutEngine, + fixtureDir, + flagValue, + hasFlag, + loadLibrary, + numberFlag, + refuseEngineFlag, + repoRoot, + resolveEngine, +} from './support.mjs'; + +refuseEngineFlag(); + +const quick = hasFlag('quick'); +const chunkSize = numberFlag('chunk', 18); +const DEFAULT_TRANSCRIPTS = ['transcript-sprint-review.json', 'transcript-giant-list.json']; +const given = flagValue('transcript', null); +const transcriptPaths = + given === null + ? DEFAULT_TRANSCRIPTS.map((f) => path.join(fixtureDir, f)) + : [path.resolve(given)]; + +const lib = loadLibrary(); +const { StreamSession, presets, projectRun, segmentRuns } = lib; + +// The view-layer half of the pipeline is not in `loadLibrary`'s namespace (that +// list is the Node-safe engine/stream/selection modules). These two modules are +// react-native-free for exactly this reason — see the note at the top of +// src/view/runIdentity.ts — so they are required straight out of dist/. +const require = createRequire(import.meta.url); +const viewDir = path.join(repoRoot, 'dist', 'view'); +const cachePath = path.join(viewDir, 'projectionCache.js'); +if (!existsSync(cachePath)) { + console.log( + '[bench:projection] dist/ predates src/view/projectionCache.ts — run `npm run build` first.', + ); + exitWithoutEngine('[bench:projection]', 'dist/ has no view/projectionCache.js to measure'); +} +const { createRunProjectionCache } = require(cachePath); +const { runKey } = require(path.join(viewDir, 'runIdentity.js')); + +const engine = await resolveEngine(lib, '[bench:projection]'); +if (!engine) exitWithoutEngine('[bench:projection]'); + +/** A pure EmbedLookup that claims nothing and sums the top-level spans it sees. */ +function meter() { + let counting = false; + let chars = 0; + return { + embed: (node, context) => { + if (counting && context.topLevel) chars += node.span.end - node.span.start; + return undefined; + }, + measure(body) { + counting = true; + try { + return body(); + } finally { + counting = false; + } + }, + chars: () => chars, + }; +} + +/** + * Replays `deltas` through a real StreamSession and runs the view pipeline on + * every commit. With `cached` false the cache is skipped entirely, which is the + * pre-fix behaviour: every run is reprojected in full on every commit that + * changed it. + */ +function replay(deltas, cached) { + const gauge = meter(); + const caches = new Map(); + const session = new StreamSession({ engine: engine.engine, options: presets.llmChat }); + let worst = 0; + let commits = 0; + let runsAtEnd = 0; + + const draw = () => { + const snapshot = session.snapshot(); + const doc = snapshot.document; + const runs = segmentRuns(doc, { + settledUntil: snapshot.settledUntil, + embed: gauge.embed, + }); + runsAtEnd = runs.length; + commits += 1; + runs.forEach((run, index) => { + if (run.standalone) return; + const unsettledTail = + snapshot.phase === 'streaming' && run.span.end > snapshot.settledUntil; + const key = runKey(run, index, runs.length, unsettledTail); + const before = gauge.chars(); + if (cached) { + let cache = caches.get(key); + if (cache === undefined) { + cache = createRunProjectionCache(); + caches.set(key, cache); + } + gauge.measure(() => cache.project(run, doc, { embed: gauge.embed })); + } else { + gauge.measure(() => projectRun(run, doc, { embed: gauge.embed })); + } + worst = Math.max(worst, gauge.chars() - before); + }); + }; + + const started = performance.now(); + for (const delta of deltas) { + session.append(delta); + draw(); + } + session.finalize(); + draw(); + const ms = performance.now() - started; + + return { + projected: gauge.chars(), + source: session.snapshot().document.source.length, + worst, + commits, + runsAtEnd, + ms, + }; +} + +/** The transcript's deltas, re-chunked to `chunkSize` so both files are read at + * the same delta granularity — amplification is a function of it. */ +function deltasOf(transcriptPath, replicas) { + const transcript = JSON.parse(readFileSync(transcriptPath, 'utf8')); + let text = ''; + for (let i = 0; i < replicas; i += 1) { + if (i > 0) text += '\n\n'; + text += transcript.deltas.join(''); + } + const out = []; + for (let at = 0; at < text.length; at += chunkSize) { + out.push(text.slice(at, at + chunkSize)); + } + return out; +} + +function fmt(n) { + return n.toLocaleString('en-US'); +} + +console.log( + `[bench:projection] engine=md4c chunk=${chunkSize} — projected characters per document character\n`, +); + +/** + * The most `cached` amplification may grow when the document doubles. + * + * Linear reuse is ~1.00 (measured: 0.99 and 1.00 on the two transcripts) and + * the pipeline this replaced is ~2.00 (measured: 1.98 and 2.09), so the + * threshold sits between them, nearer the good end: the counts are exact, so + * the only slack this needs to leave is for a transcript or `--chunk` change + * shifting how long a block spends as the redrawn tail. Anything that reaches + * 1.25 has stopped tracking the deltas and started tracking the document. + */ +const GROWTH_LIMIT = 1.25; +const overGrowth = []; + +for (const transcriptPath of transcriptPaths) { + const name = path.basename(transcriptPath); + console.log(name); + const sizes = quick ? [1] : [1, 2]; + const amps = { cached: [], full: [] }; + + for (const replicas of sizes) { + const deltas = deltasOf(transcriptPath, replicas); + for (const mode of ['cached', 'full']) { + const r = replay(deltas, mode === 'cached'); + const amp = r.projected / r.source; + amps[mode].push(amp); + console.log( + ` ${String(replicas).padStart(2)}x ${mode.padEnd(6)} ` + + `doc=${fmt(r.source).padStart(8)} projected=${fmt(r.projected).padStart(11)} ` + + `projected/doc=${amp.toFixed(1).padStart(7)} worst=${fmt(r.worst).padStart(7)} ` + + `runs=${String(r.runsAtEnd).padStart(3)} ${r.ms.toFixed(1)}ms`, + ); + } + } + + if (sizes.length > 1) { + const growth = (mode) => amps[mode][1] / amps[mode][0]; + const cachedGrowth = growth('cached'); + const over = cachedGrowth > GROWTH_LIMIT; + if (over) overGrowth.push({ name, growth: cachedGrowth }); + console.log( + ` growth ${over ? 'OVER ' : ''}cached=${cachedGrowth.toFixed(2)}x ` + + `full=${growth('full').toFixed(2)}x ` + + ` (per doubling; ~1.00 is linear, ~2.00 is quadratic; gate: cached <= ${GROWTH_LIMIT.toFixed(2)}x)`, + ); + } + console.log(''); +} + +console.log( + 'Read `cached` as the shipped pipeline and `full` as what it replaced. The gate is\n' + + `that \`cached\` growth stays at or under ${GROWTH_LIMIT.toFixed(2)}x per doubling — projection work must\n` + + 'track the deltas, not the document they accumulate into. Exceeding it exits 1.\n' + + 'docs/PERFORMANCE.md carries the invariant.\n' + + '\nA document that never anchors has no settled blocks to reuse, so both columns\n' + + 'stay high for the giant-list transcript — the reuse is real, the anchoring is\n' + + 'what it depends on. The GATE is the growth column, not the absolute\n' + + 'amplification: an unanchored stream is expensive on both pipelines by\n' + + 'construction, and gating that number would only pin the fixture.', +); + +if (overGrowth.length > 0) { + for (const { name, growth } of overGrowth) { + console.error( + `[bench:projection] ${name}: cached amplification grew ${growth.toFixed(2)}x per doubling, ` + + `over the ${GROWTH_LIMIT.toFixed(2)}x limit — incremental projection is no longer reusing ` + + 'settled blocks. Start at src/view/projectionCache.ts and runKey in src/view/runIdentity.ts.', + ); + } + process.exit(1); +} diff --git a/bench/streaming-replay.mjs b/bench/streaming-replay.mjs index 0316dd9..ce27a15 100644 --- a/bench/streaming-replay.mjs +++ b/bench/streaming-replay.mjs @@ -8,8 +8,39 @@ // - parse-input size: how many characters the engine actually read per // append (tail-only reparse means this tracks the unsettled tail, not // the accumulated document; construct-free appends skip the engine); -// - incremental-vs-full reparse ratio: total streamed append time versus -// what a naive reparse-on-every-token renderer would pay. +// - incremental-vs-full reparse ratio: what the streamed path costs versus +// what a naive reparse-on-every-token renderer would pay. BOTH SIDES ARE +// THE SAME STATISTIC — the median, across repeats, of one replay's total — +// and the run prints the two totals it divided, because a ratio whose +// numerator and denominator are computed differently is not a measurement +// of anything. (It used to divide a SUM of every append time by a MEDIAN +// full parse times the chunk count, so every append outlier — GC, a JIT +// tier-up — landed in the numerator and none in the denominator. At this +// fixture's size that is the whole signal: chunk p99 is ~30x p50.) +// +// TWO TRANSCRIPTS BY DEFAULT, AND WHY BOTH NUMBERS HAVE TO BE PUBLISHED +// --------------------------------------------------------------------- +// Tail-only reparse depends on the stream ANCHORING: a blank line closes a +// paragraph, the blocks before it freeze, and every later append parses only +// what came after. `StreamSession.isAnchorSafe` returns false for a list and +// for unclosed/indented code, and a blank line does not end a list — so the +// commonest long LLM answer shape, one bullet list, never anchors at all and +// reparses its whole accumulated text on every single append. +// +// So this bench replays two pinned transcripts and prints both: +// +// transcript-sprint-review.json headings, prose, a table, a fenced block — +// anchors constantly; parse input per append +// stays a few hundred characters no matter +// how long the stream runs. +// transcript-giant-list.json one 420-item bullet list — never anchors; +// `max/full` sits at ~1.0 and the +// incremental-vs-full ratio approaches (and +// can exceed) 1. +// +// Quoting only the first number as a property of the library is the mistake +// the second transcript exists to make impossible. `--transcript PATH` still +// narrows the run to one file. // // Per-chunk cost is the library's actual differentiator, and it has to hold up // on the engine that ships: a parser that is fast cold can still lose here if @@ -17,17 +48,48 @@ // bench/crossing.mjs). That is the pairing to read this file with. // // Usage: node bench/streaming-replay.mjs [--quick] [--transcript PATH] -// [--repeat N] [--max-chunks N] [--replicas N] +// [--repeat N] [--max-chunks N] [--replicas N] [--require-engine] +// [--budget MS] [--budget-chunk MS] [--budget-finalize MS] // // --replicas N streams the transcript N times back-to-back (separated by a // blank-line delta) as ONE growing session: the naive baseline's cost grows // with the accumulated document while tail-only parsing stays flat, so the // ratio shrinks as the stream gets longer. +// +// AND WHY THIS FILE IS ALSO A GATE +// -------------------------------- +// `bench:pathological` gates the adversarial DOCUMENT shapes; the adversarial +// STREAMING shape is this file's never-anchoring transcript, and until +// --budget existed nothing anywhere failed on it. A stream that never anchors +// re-reads its whole accumulated text on every append, so a repair or splice +// pass that stops being linear shows up here first and in `ms/chunk` — the one +// number a user feels as jank — while every document-shaped gate stays green. +// So the workflows run this file too: +// +// --budget MS the default budget for both gated numbers below. +// --budget-chunk MS p99 append latency, per transcript. p99 rather than +// the max because one GC pause in 2484 chunks is not a +// regression, and rather than p50 because the tail is +// where a stall lives. +// --budget-finalize MS the single full clean parse `finalize` does. +// +// Both are per TRANSCRIPT: a run with no --transcript replays two of them and +// each is gated on its own numbers. Without a budget nothing fails, because +// the absolute milliseconds belong to the machine and `npm run bench:streaming` +// on a laptop should not go red. +// +// --require-engine turns "the addon did not resolve" (and "StreamSession could +// not take a character") from an exit-0 report into a failure, exactly as in +// bench/pathological.mjs: a gate that exits 0 having measured nothing is worse +// than no gate, and a protocol-version drift between the built addon and dist/ +// is a likelier cause here than a missing compiler. Workflows pass it; a +// laptop with no toolchain should not. import { readFileSync } from 'node:fs'; import path from 'node:path'; import { + exitWithoutEngine, fixtureDir, fmtMs, hasFlag, @@ -39,17 +101,38 @@ import { } from './support.mjs'; const quick = hasFlag('quick'); -const transcriptPath = path.resolve( - flagValue('transcript', path.join(fixtureDir, 'transcript-sprint-review.json')), -); +// The anchoring transcript first: it is the one whose numbers the docs quote, +// and reading the never-anchoring one straight after it is the point. +const DEFAULT_TRANSCRIPTS = ['transcript-sprint-review.json', 'transcript-giant-list.json']; +const given = flagValue('transcript', null); +const transcriptPaths = + given === null + ? DEFAULT_TRANSCRIPTS.map((f) => path.join(fixtureDir, f)) + : [path.resolve(given)]; const repeat = numberFlag('repeat', quick ? 1 : 3); const maxChunks = numberFlag('max-chunks', quick ? 150 : Infinity); +const replicas = numberFlag('replicas', 1); + +// The two gated numbers, resolved before anything runs so the header line can +// print what is being gated — a threshold only visible by reading the source is +// one nobody re-tunes when the numbers move. +const budgetMs = numberFlag('budget', undefined); +const chunkBudgetMs = numberFlag('budget-chunk', budgetMs); +const finalizeBudgetMs = numberFlag('budget-finalize', budgetMs); +const gating = chunkBudgetMs !== undefined || finalizeBudgetMs !== undefined; + +let anyOver = false; +// A budget over a replay that measured no chunk (`--max-chunks 0`, an empty +// transcript) is vacuous, which is the same failure as --require-engine +// catching an unresolvable addon. Tracked apart from `anyOver` so the closing +// message can say which of the two happened. +let anyVacuous = false; const lib = loadLibrary(); const { StreamSession, parseDocument, presets, visit } = lib; const engine = await resolveEngine(lib, '[bench:streaming]'); -if (!engine) process.exit(0); +if (!engine) exitWithoutEngine('[bench:streaming]'); // StreamSession gets its own probe on top of the engine probe resolveEngine // already did: a session that cannot take even one character is a different @@ -61,16 +144,7 @@ try { probe.finalize(); } catch (err) { console.log(`[bench:streaming] StreamSession unavailable (${err.message}) — nothing to measure yet.`); - process.exit(0); -} - -const replicas = numberFlag('replicas', 1); -const transcript = JSON.parse(readFileSync(transcriptPath, 'utf8')); -const baseDeltas = transcript.deltas.slice(0, maxChunks); -const deltas = []; -for (let i = 0; i < replicas; i++) { - if (i > 0) deltas.push('\n\n'); - deltas.push(...baseDeltas); + exitWithoutEngine('[bench:streaming]', `StreamSession threw on a one-character append (${err.message})`); } function collectNodes(doc) { @@ -81,133 +155,278 @@ function collectNodes(doc) { return seen; } -const out = { - chunkTimes: [], - changedCounts: [], - // Parse-input sizes for the LAST replay: appends only (finalize's single - // full clean parse is reported separately). - parseInputs: [], - finalizeInput: 0, - finalizeMs: 0, - totalMs: 0, - finalSnapshot: null, -}; - /** - * One full replay of the transcript through a fresh session, timing each - * append. The engine is wrapped so the bench can see how many characters the - * splice actually handed the parser — that count, not the accumulated - * document length, is what tail-only reparse is supposed to keep small. + * Replays one transcript and prints its block of numbers. + * + * Everything is per-transcript state: each file gets its own fresh sessions, + * its own warmup and its own naive baseline, so no transcript's numbers are + * measured on a heap the previous one shaped. */ -function replay(record) { - const inputs = []; - const recordingEngine = { - name: `recording(${engine.name})`, - parse(source, options) { - inputs.push(source.length); - return engine.engine.parse(source, options); - }, +function runTranscript(transcriptPath) { + const transcript = JSON.parse(readFileSync(transcriptPath, 'utf8')); + const baseDeltas = transcript.deltas.slice(0, maxChunks); + const deltas = []; + for (let i = 0; i < replicas; i++) { + if (i > 0) deltas.push('\n\n'); + deltas.push(...baseDeltas); + } + + const out = { + chunkTimes: [], + // One entry per timed replay: the sum of that replay's append times. The + // ratio below is the median of these, against the median of the naive + // baseline's per-replay totals — same shape, same outlier exposure. + replayTotals: [], + changedCounts: [], + // Parse-input sizes for the LAST replay: appends only (finalize's single + // full clean parse is reported separately). + parseInputs: [], + finalizeInput: 0, + finalizeMs: 0, + totalMs: 0, + finalSnapshot: null, }; - const session = new StreamSession({ - engine: recordingEngine, - options: presets.llmChat, - }); - let previousNodes = new Set(); - const runStart = performance.now(); - - for (const delta of deltas) { - const t0 = performance.now(); - session.append(delta); - const ms = performance.now() - t0; - - const doc = session.snapshot().document; - const nodes = collectNodes(doc); - let changed = 0; - for (const node of nodes) { - if (!previousNodes.has(node)) changed += 1; + + /** + * One full replay of the transcript through a fresh session, timing each + * append. The engine is wrapped so the bench can see how many characters the + * splice actually handed the parser — that count, not the accumulated + * document length, is what tail-only reparse is supposed to keep small. + */ + const replay = (record) => { + const inputs = []; + const recordingEngine = { + name: `recording(${engine.name})`, + parse(source, options) { + inputs.push(source.length); + return engine.engine.parse(source, options); + }, + }; + const session = new StreamSession({ + engine: recordingEngine, + options: presets.llmChat, + }); + let previousNodes = new Set(); + const runStart = performance.now(); + let appendTotal = 0; + + for (const delta of deltas) { + const t0 = performance.now(); + session.append(delta); + const ms = performance.now() - t0; + appendTotal += ms; + + const doc = session.snapshot().document; + const nodes = collectNodes(doc); + let changed = 0; + for (const node of nodes) { + if (!previousNodes.has(node)) changed += 1; + } + previousNodes = nodes; + if (record) { + out.chunkTimes.push(ms); + out.changedCounts.push(changed); + } } - previousNodes = nodes; + + const appendInputs = inputs.slice(); + const f0 = performance.now(); + session.finalize('end'); + const finalizeMs = performance.now() - f0; if (record) { - out.chunkTimes.push(ms); - out.changedCounts.push(changed); + out.replayTotals.push(appendTotal); + out.parseInputs = appendInputs; + out.finalizeMs = finalizeMs; + out.totalMs = performance.now() - runStart; + out.finalSnapshot = session.snapshot(); + out.finalizeInput = inputs.length > appendInputs.length ? inputs[inputs.length - 1] : 0; } - } + }; + + // Warmup replay (untimed) so the session path is JIT-compiled before + // measurement, matching the warmup the naive-baseline loop gets below. + replay(false); + + for (let r = 0; r < repeat; r += 1) replay(true); + + const totalChars = deltas.reduce((acc, d) => acc + d.length, 0); + const effectiveAppends = deltas.filter((d) => d.length > 0).length; + + // Resolved once, outside every timed region: reading it out of the snapshot + // inside the loop would put a property walk inside the measurement. + const finalSource = out.finalSnapshot ? out.finalSnapshot.document.source : deltas.join(''); - const appendInputs = inputs.slice(); - const f0 = performance.now(); - session.finalize('end'); - const finalizeMs = performance.now() - f0; - if (record) { - out.parseInputs = appendInputs; - out.finalizeMs = finalizeMs; - out.totalMs = performance.now() - runStart; - out.finalSnapshot = session.snapshot(); - out.finalizeInput = inputs.length > appendInputs.length ? inputs[inputs.length - 1] : 0; + // Incremental-vs-full reparse ratio: what the streamed path costs against + // what a naive reparse-on-every-token renderer would pay. Lower is better; + // 1.0 means no win over naive reparse. With tail-only reparse + the + // construct-free fast path this sits far below 1 and shrinks as documents + // grow — on a transcript that ANCHORS. On one that never anchors it climbs + // towards 1 and can pass it, because every append reparses the whole document + // and pays the splice on top. The baseline is measured with the SAME engine + // the streamed numbers came from, so what the ratio isolates is the + // incremental strategy and nothing else — a naive loop timed on some other + // parser would just be a parser comparison wearing a different name. + // + // MATCHED STATISTICS, WHICH IS WHY THE BASELINE IS A LOOP AND NOT A CONSTANT. + // The naive side used to be `median(a few full parses) × chunk count`, while + // the streamed side was the SUM of every append. A sum carries its outliers + // and a median discards them, so the ratio was a measurement of this + // machine's noise as much as of the library: on the 1.2 kB transcript chunk + // p99 is ~30x p50, and repeated runs here swung the printed figure across + // 1.0 in both directions. So the baseline now runs `repeat` REPLAYS of its + // own — each one the full `chunk count` reparses, summed exactly the way the + // streamed replay sums its appends — and the ratio divides the median of one + // side's per-replay totals by the median of the other's. Same estimator, same + // outlier exposure, same number of samples. + // + // It costs what it measures: the baseline is now the same order of work as + // the streamed side (that is the point of the comparison), where the old + // version was ten parses. That is the price of a number that means something. + // + // The naive renderer is modelled as reparsing the FINAL document on every + // chunk rather than the accumulated prefix, which overstates it by roughly + // the average prefix fraction. That approximation is unchanged, and it is + // stated in the printed lines so nobody has to read this comment to know + // what was divided. + const naiveTotals = []; + // One untimed warmup REPLAY, not one untimed parse: the streamed side gets a + // whole untimed replay above, and warming the two sides by different amounts + // is the same asymmetry in a different place — it left the naive side's + // first timed replay carrying the JIT tier-up for all of them. + for (let i = 0; i < deltas.length; i += 1) { + parseDocument(finalSource, presets.llmChat, engine.engine); + } + for (let r = 0; r < repeat; r += 1) { + let total = 0; + for (let i = 0; i < deltas.length; i += 1) { + const t0 = performance.now(); + parseDocument(finalSource, presets.llmChat, engine.engine); + total += performance.now() - t0; + } + naiveTotals.push(total); } -} -// Warmup replay (untimed) so the session path is JIT-compiled before -// measurement, matching the warmup the naive-baseline loop gets below. -replay(false); - -for (let r = 0; r < repeat; r += 1) replay(true); - -const totalChars = deltas.reduce((acc, d) => acc + d.length, 0); -const effectiveAppends = deltas.filter((d) => d.length > 0).length; - -// Resolved once, outside every timed region: reading it out of the snapshot -// inside the loop would put a property walk inside the measurement. -const finalSource = out.finalSnapshot ? out.finalSnapshot.document.source : deltas.join(''); - -// Incremental-vs-full reparse ratio: total streamed append time versus what a -// naive reparse-on-every-token renderer would pay (chunks × full reparse of -// the final document). Lower is better; 1.0 means no win over naive reparse. -// With tail-only reparse + the construct-free fast path this should sit far -// below 1 and shrink as documents grow. The baseline is measured with the -// SAME engine the streamed numbers came from, so what the ratio isolates is -// the incremental strategy and nothing else — a naive loop timed on some -// other parser would just be a parser comparison wearing a different name. -const fullRuns = quick ? 3 : 10; -const fullTimes = []; -parseDocument(finalSource, presets.llmChat, engine.engine); // warmup -for (let i = 0; i < fullRuns; i += 1) { - const t0 = performance.now(); - parseDocument(finalSource, presets.llmChat, engine.engine); - fullTimes.push(performance.now() - t0); -} + const replicaNote = replicas > 1 ? ` × ${replicas} replicas` : ''; + console.log(`streaming replay: ${transcript.name ?? path.basename(transcriptPath)}${replicaNote}`); + console.log(` chunks: ${deltas.length} (${totalChars} UTF-16 units), ${repeat} replay(s)${quick ? ' [quick]' : ''}`); -const replicaNote = replicas > 1 ? ` × ${replicas} replicas` : ''; -console.log(`streaming replay: ${transcript.name ?? path.basename(transcriptPath)}${replicaNote}`); -console.log(` chunks: ${deltas.length} (${totalChars} UTF-16 units), ${repeat} replay(s)${quick ? ' [quick]' : ''}`); - -const t = stats(out.chunkTimes); -const c = stats(out.changedCounts); -const fullMs = stats(fullTimes).p50; -const streamedAppendMs = out.chunkTimes.reduce((acc, ms) => acc + ms, 0) / repeat; -const naiveMs = fullMs * deltas.length; -const reparseRatio = naiveMs > 0 ? streamedAppendMs / naiveMs : NaN; - -console.log(` engine: ${engine.name}`); -console.log(` ms/chunk: p50 ${fmtMs(t.p50)} | p95 ${fmtMs(t.p95)} | p99 ${fmtMs(t.p99)} | max ${fmtMs(t.max)}`); -console.log( - ` changed-identity nodes/chunk: p50 ${c.p50} | p95 ${c.p95} | p99 ${c.p99} | max ${c.max} | mean ${c.mean.toFixed(1)}`, -); -console.log(` last replay: ${fmtMs(out.totalMs)} total, finalize ${fmtMs(out.finalizeMs)}`); -if (out.parseInputs.length > 0) { - const pi = stats(out.parseInputs); - const fastPathAppends = effectiveAppends - out.parseInputs.length; + const t = stats(out.chunkTimes); + const c = stats(out.changedCounts); + const streamed = stats(out.replayTotals); + const naive = stats(naiveTotals); + const reparseRatio = naive.p50 > 0 ? streamed.p50 / naive.p50 : NaN; + + console.log(` engine: ${engine.name}`); + console.log(` ms/chunk: p50 ${fmtMs(t.p50)} | p95 ${fmtMs(t.p95)} | p99 ${fmtMs(t.p99)} | max ${fmtMs(t.max)}`); + console.log( + ` changed-identity nodes/chunk: p50 ${c.p50} | p95 ${c.p95} | p99 ${c.p99} | max ${c.max} | mean ${c.mean.toFixed(1)}`, + ); + console.log(` last replay: ${fmtMs(out.totalMs)} total, finalize ${fmtMs(out.finalizeMs)}`); + if (out.parseInputs.length > 0) { + const pi = stats(out.parseInputs); + const fastPathAppends = effectiveAppends - out.parseInputs.length; + console.log( + ` parse input/append: mean ${pi.mean.toFixed(0)} | p95 ${pi.p95} | max ${pi.max} of ${finalSource.length} final chars (max/full = ${(pi.max / finalSource.length).toFixed(3)})`, + ); + console.log( + ` engine calls: ${out.parseInputs.length}/${effectiveAppends} appends (${fastPathAppends} construct-free appends skipped the engine); finalize parsed ${out.finalizeInput} chars once`, + ); + // Said in words, not left to the reader to spot: a max parse input equal + // to the whole document means the anchor never moved during the stream — + // the tail-only story does not hold for this shape, and the mean tells you + // how much of the document the average append re-read. `settledUntil` on + // the final snapshot cannot say this, because finalize settles everything. + if (pi.max / finalSource.length > 0.9) { + console.log( + ` NEVER ANCHORED: the largest append re-read ${((pi.max / finalSource.length) * 100).toFixed(1)}% of the ` + + `final document and the average one ${((pi.mean / finalSource.length) * 100).toFixed(1)}%. ` + + 'A list (or an unclosed fence, or one giant paragraph) offers the session no safe anchor.', + ); + } + } + // Every term of the ratio is printed, because "0.435" on its own is a number + // nobody can check and two docs already managed to quote it in opposite + // directions. The spread line is the honest caveat: where the two bands + // overlap, the ratio is inside the noise and no ×1 reading of it is safe. + console.log( + ` incremental-vs-full reparse ratio: ${reparseRatio.toFixed(3)} ` + + '(lower is better; 1.0 = no cheaper than a full reparse per chunk)', + ); console.log( - ` parse input/append: mean ${pi.mean.toFixed(0)} | p95 ${pi.p95} | max ${pi.max} of ${finalSource.length} final chars (max/full = ${(pi.max / finalSource.length).toFixed(3)})`, + ` streamed ${fmtMs(streamed.p50)} = median of ${streamed.n} replay(s), each the SUM of its ` + + `${deltas.length} append times`, ); console.log( - ` engine calls: ${out.parseInputs.length}/${effectiveAppends} appends (${fastPathAppends} construct-free appends skipped the engine); finalize parsed ${out.finalizeInput} chars once`, + ` naive ${fmtMs(naive.p50)} = median of ${naive.n} replay(s), each the SUM of ` + + `${deltas.length} full reparses of the ${finalSource.length}-char final document`, ); + if (streamed.n > 1 || naive.n > 1) { + console.log( + ` spread streamed ${fmtMs(streamed.min)}–${fmtMs(streamed.max)}, ` + + `naive ${fmtMs(naive.min)}–${fmtMs(naive.max)} ` + + `(ratio ${(streamed.min / naive.max).toFixed(3)}–${(streamed.max / naive.min).toFixed(3)} at the extremes)`, + ); + } + if (out.finalSnapshot) { + const s = out.finalSnapshot; + console.log( + ` final: phase=${s.phase}, blocks=${s.document.blocks.length}, settledUntil=${s.settledUntil}/${s.document.source.length}`, + ); + } + + // ---- the gate ----------------------------------------------------------- + // + // Printed only when a budget was passed, so a plain `npm run bench:streaming` + // stays a report. `ok`/`OVER` and the same padding as bench/pathological.mjs, + // because the two are read (and grepped) together in a workflow log. + if (!gating) return; + + const gate = (label, ms, budget) => { + if (budget === undefined) { + console.log(` ${'off'.padEnd(5)} ${label.padEnd(10)} ${fmtMs(ms)} (no budget passed for this one)`); + return; + } + const over = ms > budget; + if (over) anyOver = true; + console.log(` ${(over ? 'OVER' : 'ok').padEnd(5)} ${label.padEnd(10)} ${fmtMs(ms)} vs ${budget} ms`); + }; + + // A budget over a replay that timed nothing passes over zero samples, which + // is the failure --require-engine exists to stop one line further up. Both + // ways in are reachable from flags alone: `--max-chunks 0` leaves no append + // to time, `--repeat 0` leaves no recorded replay at all. + if (out.chunkTimes.length === 0 || out.replayTotals.length === 0) { + anyVacuous = true; + console.log( + ` ${'none'.padEnd(5)} ${'gate'.padEnd(10)} nothing was timed — ` + + `\`--max-chunks ${maxChunks}\` left ${deltas.length} chunk(s) and \`--repeat ${repeat}\` ` + + `${out.replayTotals.length} recorded replay(s)`, + ); + return; + } + + gate('chunk p99', t.p99, chunkBudgetMs); + gate('finalize', out.finalizeMs, finalizeBudgetMs); } -console.log( - ` incremental-vs-full reparse ratio: ${reparseRatio.toFixed(3)} (streamed ${fmtMs(streamedAppendMs)} vs naive ${deltas.length} × ${fmtMs(fullMs)} = ${fmtMs(naiveMs)}; lower is better)`, -); -if (out.finalSnapshot) { + +if (gating) { + console.log( + `gating each transcript: chunk p99 ${chunkBudgetMs === undefined ? 'off' : `${chunkBudgetMs} ms`}, ` + + `finalize ${finalizeBudgetMs === undefined ? 'off' : `${finalizeBudgetMs} ms`}`, + ); +} + +for (const [i, transcriptPath] of transcriptPaths.entries()) { + if (i > 0) console.log(''); + runTranscript(transcriptPath); +} + +if (gating && (anyOver || anyVacuous)) { + console.log(''); console.log( - ` final: phase=${out.finalSnapshot.phase}, blocks=${out.finalSnapshot.document.blocks.length}, settledUntil=${out.finalSnapshot.settledUntil}/${out.finalSnapshot.document.source.length}`, + anyVacuous && !anyOver + ? 'a replay timed nothing — a gate over nothing is a failure.' + : 'budget exceeded — failing.', ); + process.exit(1); } diff --git a/bench/support.mjs b/bench/support.mjs index fe7afa9..3a3d724 100644 --- a/bench/support.mjs +++ b/bench/support.mjs @@ -68,6 +68,16 @@ export function loadLibrary() { ...require(path.join(dist, 'engine', 'native', 'index.js')), ...require(path.join(dist, 'stream', 'StreamSession.js')), ...require(path.join(dist, 'document', 'visit.js')), + // The rest of the pipeline a consumer runs on every snapshot, and the + // only part of it that is pure TypeScript over the parsed document: + // tail repair, run segmentation and run projection. They are here rather + // than in one bench because `bench:pathological` times them as the stages + // after the parse (the parse is the linear one; these are not), and a + // second copy of the require list is how the two drift apart. All three + // are react-native-free, so requiring them cannot fail in plain Node. + ...require(path.join(dist, 'stream', 'repair.js')), + ...require(path.join(dist, 'selection', 'runs.js')), + ...require(path.join(dist, 'selection', 'mapSelection.js')), }; } @@ -193,6 +203,36 @@ export async function resolveEngine(lib, label) { return { name: 'native', engine: native.engine, parse: native.parse, addonPath: native.addonPath }; } +/** + * What a bench does when `resolveEngine` (or any other precondition) came back + * empty: exit 0 after reporting, or exit 1 when the caller passed + * `--require-engine`. + * + * WHY THE FLAG EXISTS. Exiting 0 is right for `npm run bench:*` on a laptop + * with no C++ toolchain — "this machine cannot build the addon" is not a + * regression, and a red job for it teaches people to ignore the job. It is + * exactly wrong for a CI step that is a GATE: `bench:pathological --budget` + * and `bench:projection` are supposed to fail on a cliff, and a step that + * exits 0 having measured nothing is a green gate over zero samples. The + * likeliest cause is not a missing compiler at all but a protocol-version + * drift between the built addon and dist/, which `resolveEngine` reports and + * then swallows. + * + * So the workflows pass `--require-engine` and developers do not. `label` + * prefixes the message, e.g. `[bench:pathological]`. + */ +export function exitWithoutEngine(label, what = 'the native engine did not resolve') { + if (hasFlag('require-engine')) { + console.error( + `${label} --require-engine was passed and ${what}, so this run measured nothing. ` + + 'Failing rather than reporting a gate that passed over zero samples.', + ); + process.exit(1); + } + console.log(`${label} exiting 0 (pass --require-engine to make this a failure).`); + process.exit(0); +} + /** * Times `run()` `iterations` times after `warmup` untimed calls. * diff --git a/bench/throughput.mjs b/bench/throughput.mjs index 61223bf..95589f0 100644 --- a/bench/throughput.mjs +++ b/bench/throughput.mjs @@ -1,7 +1,16 @@ #!/usr/bin/env node -// Cold-parse throughput: parse a concatenated markdown corpus N times and -// report MB/s. Corpus = every CommonMark spec example's markdown plus the -// conformance fixtures, replicated to a workload-sized document. +// Full-document parse throughput, WARM: parse a concatenated markdown corpus +// N times after 3 untimed warmup passes (1 with --quick) and report MB/s. +// Corpus = every CommonMark spec example's markdown plus the conformance +// fixtures, replicated to a workload-sized document. +// +// Warm, not cold, and the distinction is not pedantry: `bench:pathological` is +// the no-warmup bench ("cold" everywhere in docs/BENCHMARKS.md means exactly +// that), and on this repo's own numbers the two readings of the same input +// differ by ~7×. Quoting an MB/s from here against another engine's genuinely +// cold figure compares the wrong things. Steady state is the right question +// for a whole-corpus parse — it is what a sustained workload sees — which is +// why the warmup is here at all. // // Usage: node bench/throughput.mjs [--quick] [--iterations N] [--replicas R] // @@ -53,7 +62,10 @@ const s = stats(samples); const mbPerSecMean = bytes / 1e6 / (s.mean / 1000); const mbPerSecBest = bytes / 1e6 / (s.min / 1000); -console.log('parse throughput (preset: llmChat)'); +// "warm" in the heading rather than only in the `+N warmup` line below: this +// heading is the line that gets copied into a doc, and the row it labels was +// mislabelled "Cold parse" for exactly that long. +console.log('parse throughput, warm (preset: llmChat)'); console.log(` corpus: ${fmtBytes(bytes)} (${corpus.length} UTF-16 units)`); console.log(` iterations: ${iterations} (+${warmup} warmup)${quick ? ' [quick]' : ''}`); console.log( diff --git a/conformance/fixtures/transcript-giant-list.json b/conformance/fixtures/transcript-giant-list.json new file mode 100644 index 0000000..483d6aa --- /dev/null +++ b/conformance/fixtures/transcript-giant-list.json @@ -0,0 +1,2490 @@ +{ + "name": "giant-bullet-list", + "description": "Deterministic replay transcript: a 420-item single bullet list — the commonest long LLM answer shape, and one that never anchors, because a blank line does not end a list. Chunked into LLM-ish deltas (0-18 UTF-16 units, BMP-only so every cut is surrogate-safe). Pinned so bench/streaming-replay.mjs reports the never-anchoring case next to the anchoring one.", + "deltas": [ + "## Chec", + "klist for the m", + "o", + "bile per", + "f", + "ormance pass\n\n- Ca", + "che **toke", + "n r", + "efresh** with", + "out a ", + "re-render (", + "", + "#1", + ")", + "\n- Trim **cold st", + "art** i", + "n t", + "he re", + "", + "le", + "as", + "e build (#2)\n-", + " Pin r", + "etry", + " backoff on th", + "e JS thread onl", + "y (#3)", + "\n- Debounce **lay", + "", + "out th", + "rash*", + "* behind a featu", + "re flag (#4)\n- De", + "fer push token", + "s in t", + "he release buil", + "d (#5)\n-", + " Batch tel", + "emetr", + "y", + " sampling per navi", + "gatio", + "n entry ", + "(", + "#6)\n- ", + "Audit schema m", + "igration once ", + "per session", + " (#7)\n- Pin ", + "lay", + "out thrash an", + "d log the delta (", + "#8)\n- Trim of", + "fline queue un", + "d", + "e", + "r ", + "a memory c", + "ap ", + "(", + "#9)\n- Pre", + "fetch `dark mod", + "e", + " tokens", + "` without a re-", + "rend", + "er (#10)\n- D", + "e", + "bounce `schema m", + "igration` on the ", + "JS thread on", + "ly (#11)\n- Def", + "er `cl", + "ipboard po", + "licy` without a", + " re-ren", + "der (#1", + "2)\n-", + " Batch cold st", + "", + "art with a 200 m", + "", + "s budget (#", + "13)", + "\n- C", + "ache `cli", + "pboard policy` ", + "", + "i", + "n the releas", + "e build (", + "#14)\n- Cache ", + "layout thrash per", + " navigation entry", + " (#15", + ")\n- Cache bun", + "dl", + "e size on the JS", + " th", + "read", + " only (#16)\n- Bat", + "ch image", + " c", + "ach", + "e pe", + "r na", + "vigation entry", + " (#17)\n- Measure", + " push token", + "s b", + "efore the first f", + "rame (#18)\n- Debo", + "u", + "nce c", + "rash repo", + "rting", + " before the f", + "irst frame (", + "#1", + "9)\n- Trim **push ", + "tokens** behind a", + " featur", + "e flag (#20)\n", + "- Prefetch", + " **text", + " selection** pe", + "r nav", + "igation entr", + "y (#21)\n- Debou", + "nce accessibi", + "", + "lity labels unde", + "r a memory cap (#2", + "2", + ")\n- Measure *", + "*cold start** and", + " log ", + "the ", + "d", + "e", + "lt", + "a (#23)\n- Batch", + " push", + " tokens wi", + "tho", + "ut a", + "", + " re-ren", + "der (#24)\n- ", + "Batch ", + "**deep links** wit", + "h a 2", + "00 ms ", + "bud", + "get", + " (#25)\n-", + " Measure offline", + " queue withou", + "t a re-render (#", + "26)\n- ", + "Pin tele", + "metry sampling w", + "ith", + " ", + "a 20", + "0 m", + "s bud", + "get", + " (#27)", + "\n- Audit `push tok", + "ens` per navigatio", + "n entry (#28)\n- ", + "Pin co", + "ld start under a m", + "emo", + "ry ", + "cap (#29)\n- C", + "ache `bundl", + "e size` ", + "without ", + "a r", + "e-ren", + "der (#30)\n- ", + "A", + "udit `retry ", + "backof", + "f` witho", + "u", + "t a r", + "e-render (#31)\n- I", + "nline ", + "**retry backoff*", + "* before the fi", + "rst fr", + "a", + "me (#32)\n- P", + "refet", + "ch", + " `deep links` ", + "in the", + " release build", + " (#33)\n-", + " Inline dark", + " mode t", + "okens b", + "ehind a feature f", + "lag ", + "(#34)\n- Pref", + "etch toke", + "n refre", + "sh on the JS ", + "thread on", + "ly (#35)\n-", + " Cache ", + "gesture c", + "on", + "fli", + "cts without a r", + "", + "e-render (#", + "36", + ")\n- Debounce push ", + "tokens without ", + "a re-rend", + "er ", + "(#37)\n- M", + "easure gestu", + "re con", + "flicts u", + "nder ", + "a m", + "emory ca", + "p (#38)\n- ", + "Audit *", + "*layout th", + "ra", + "sh** wit", + "", + "h a 200 m", + "s budget (", + "#39)\n- Pin ", + "", + "retry", + " backo", + "f", + "f on the JS thread", + " only ", + "(#40)", + "\n- Prefetc", + "h pus", + "h tokens and ", + "log the", + " delta (", + "#41)\n- Pre", + "fe", + "t", + "ch crash report", + "in", + "g on the JS t", + "hrea", + "d ", + "only (#", + "42)\n- Defer im", + "age cache w", + "ithou", + "t a re-", + "r", + "ender (#43", + ")\n- A", + "udit retry ", + "b", + "ackoff under a mem", + "ory cap (#44)\n- In", + "line accessi", + "bi", + "lity", + " la", + "be", + "l", + "s", + " before the ", + "first f", + "rame", + " (#45)\n- Cache sch", + "ema migration ", + "and log t", + "he delt", + "a (#46)", + "\n", + "- Cache", + " locale fall", + "bac", + "ks in the", + " release ", + "build (#47)\n- Inl", + "ine **l", + "ay", + "out thrash** wi", + "thout a re-", + "render (#4", + "8)\n- Audit ima", + "ge cache on the", + " JS th", + "read only (#49)\n- ", + "Meas", + "ure **crash repor", + "ting** an", + "d log the d", + "elta (#50)\n", + "- Meas", + "ure locale fallbac", + "ks in the release", + " ", + "build (#51)", + "\n- Cache bu", + "n", + "dle ", + "size under ", + "a memory ca", + "p ", + "(#52)\n-", + "", + " Audit clipboard", + " poli", + "cy behind ", + "a feature", + " flag (#53)\n- Tr", + "im offli", + "", + "ne que", + "ue without a ", + "re-ren", + "der (#54)\n- Pin ", + "layout", + " ", + "thra", + "s", + "h on th", + "e JS", + " thread ", + "only (#55)\n- Pi", + "n push toke", + "ns and log the ", + "delta", + " (#5", + "6)\n- Inline s", + "chema migration", + " without ", + "a re-", + "ren", + "der (#57)\n- Aud", + "it retry backo", + "ff before the fi", + "rst fra", + "me ", + "(#58)\n- Debounce ", + "crash re", + "porting per navi", + "gation entry (#59", + ")\n- Deb", + "ounce sch", + "ema mig", + "rati", + "on under ", + "a memory cap (#60", + ")\n- Batch pus", + "h token", + "s on the JS thr", + "ead only (#61)\n- ", + "Trim o", + "ffline queue wi", + "thout a ", + "re", + "-re", + "nder (#62)\n- Pin ", + "retry backo", + "ff", + " bef", + "ore the first f", + "rame (", + "#63)\n- Trim **", + "cold start** ", + "once per session ", + "(#64)", + "\n- Defer r", + "etry ", + "backoff witho", + "ut a re-", + "rend", + "er (#65)\n- Prefetc", + "h clipboard polic", + "y and log th", + "e ", + "", + "delta (#66)\n- Pr", + "efetch f", + "ont loading onc", + "e per sess", + "ion (#67)\n- ", + "Measure layout ", + "thrash in ", + "the release ", + "buil", + "d (#", + "68)\n- Inline ", + "dark mod", + "e tokens o", + "n the", + " JS thread onl", + "y (#69)\n-", + " Inline **te", + "lemetry sampling", + "** per n", + "a", + "vigation ent", + "ry (#70)\n", + "- Batc", + "h schema migration", + " behind a featu", + "re flag (#71)\n-", + " Pin imag", + "e cache", + " b", + "efore the firs", + "t fra", + "me (", + "#72)\n- Batch t", + "oken refresh w", + "ith a 200 ms budg", + "et (#73)", + "\n- I", + "nline c", + "lipb", + "oard po", + "licy with a 200 ms", + " budget (#74)\n- ", + "Defer *", + "*layout thrash", + "** per", + " na", + "vi", + "gation", + " entry (#75)\n", + "- Batch `te", + "lemetry samp", + "ling` in the r", + "elease build (#", + "7", + "6)\n- Inlin", + "e p", + "us", + "h tokens wi", + "th a 200 m", + "", + "s budget (#77)\n", + "- Measure offline", + " ", + "q", + "ueue u", + "nder ", + "", + "a memory ca", + "p (#78)\n- Deboun", + "ce push tok", + "ens before the f", + "irst f", + "rame ", + "", + "(#79)\n- Defe", + "r **crash repo", + "rting** u", + "nder a", + " memory", + " cap (#80)", + "", + "\n- D", + "eboun", + "ce **schema mi", + "gration** once pe", + "r", + " session (#81)\n- ", + "Prefetch **accessi", + "bili", + "ty labels** in th", + "e release build (#", + "82)", + "", + "\n-", + " Cach", + "e cl", + "ipboard polic", + "y on the JS thr", + "ead o", + "nl", + "y (#83)\n- Deb", + "ou", + "nce layout thras", + "h", + " be", + "hind a featur", + "e f", + "lag (#84)\n- Prefe", + "", + "tch list ", + "", + "virtua", + "lization wit", + "h", + " ", + "a 20", + "0 ms", + " budget (#85", + ")\n- Trim retry", + " backoff once per ", + "session (#86)\n- P", + "refetch gesture co", + "nflicts before the", + " first frame (#87)", + "\n- Inli", + "ne access", + "ibility labels ", + "without a re", + "-render", + " (#8", + "8)\n- Measure fon", + "t loadin", + "g p", + "er navigation", + " e", + "ntry (#8", + "", + "9)\n- T", + "rim clipboard ", + "pol", + "icy u", + "n", + "der a memo", + "ry cap (#90)\n- D", + "e", + "fer clipboard", + " pol", + "icy before the", + " first ", + "", + "frame", + " (#9", + "1)\n- Audit lay", + "out thrash o", + "n the JS thread o", + "", + "nly (#92)\n-", + " Debounce ", + "font loa", + "ding in the r", + "elease build (#9", + "3)\n- Cach", + "e offline queue b", + "", + "efore ", + "the", + " f", + "irst fra", + "me (", + "#94)\n-", + " Trim i", + "mage ", + "cache o", + "nce per sess", + "ion (", + "#95)\n- Inli", + "ne `deep links` ", + "beh", + "ind a feat", + "ure flag", + " (#", + "96)\n- Measure d", + "eep links withou", + "t a", + " re-render (#97)", + "\n- Inlin", + "e **toke", + "n refresh** befo", + "re the f", + "irst frame (", + "#98)\n- Prefetch `", + "deep links` once ", + "per session (#9", + "", + "9", + ")\n- Trim **re", + "try", + " backoff** and lo", + "g ", + "the de", + "lta (#100)\n- Pr", + "efetch im", + "age cache", + "", + " on the JS threa", + "d only (#101)\n", + "- Batc", + "h clipbo", + "ard policy with", + " a 200 ", + "ms budget (#", + "10", + "2)\n- Pref", + "etch fo", + "nt loading ", + "bef", + "ore the", + " first fram", + "e (#103)\n", + "- Defer **image", + " cac", + "he** per ", + "navigation entry ", + "(", + "#104)\n- Def", + "er ", + "pus", + "h tokens b", + "ehin", + "d a", + " feature flag ", + "(#105)\n- Trim", + " ", + "token ref", + "", + "resh with a 200 ms", + " bud", + "ge", + "t (#106)\n- Deboun", + "ce ges", + "ture conflicts und", + "er ", + "a memory c", + "ap ", + "(#107)", + "\n- Debounce `l", + "ist virtualizati", + "on` bef", + "ore", + " the fi", + "rst frame (#1", + "08", + ")\n- ", + "Me", + "asure ", + "push ", + "tokens ", + "wit", + "h a 20", + "0", + "", + " ms b", + "udget (", + "#109)\n- B", + "atch ", + "**telemetry s", + "ampling**", + " under ", + "a memory cap (#", + "110)", + "\n- Defer *", + "*deep l", + "inks** with", + " a 200 m", + "s budget (#111)\n-", + " Pin **l", + "ocale fallbacks", + "** in the relea", + "se build", + " (#112)", + "\n- Pi", + "n imag", + "e cache w", + "ith a 200 ms budge", + "t (#113)\n- Prefe", + "tch retry backoff", + " ", + "on the JS ", + "thread only (#1", + "14)\n- Cache telem", + "", + "et", + "ry sampling in", + " the release ", + "build (#115)\n- ", + "Pin dar", + "k mode tokens with", + " a 200", + " ms", + " budget (#116)\n", + "- ", + "Batch telemetry", + " sampling once p", + "er session (#117)", + "\n- Measure `", + "offline queue` per", + " navigation en", + "try (#118)\n-", + " Measu", + "re `locale fal", + "lbacks`", + " under a m", + "emory cap", + " (#119)\n- Debou", + "nce layout t", + "hrash behind a f", + "eature f", + "lag (#120", + ")\n- ", + "Batch font loadi", + "ng", + " without a re-re", + "nder ", + "(#121", + ")\n- Defer *", + "*dark mode toke", + "ns** behind a f", + "eature flag ", + "(#122)", + "\n- Cache ", + "**image cache** ", + "once per sessi", + "on (#123)\n- Me", + "asure **offlin", + "e queue** and", + " log the del", + "ta (#124)\n-", + " Aud", + "it offline qu", + "eue bef", + "ore the first ", + "frame (#125)\n", + "", + "- Pre", + "fet", + "c", + "h `image ca", + "che` in", + " the release buil", + "d (#126)\n- Pre", + "fetch list virt", + "ualization on", + " the JS t", + "hread only ", + "(#127)\n- ", + "Defer l", + "ist virtualization", + " ", + "u", + "nder a memory ", + "cap (#128)\n- P", + "refetc", + "h layout thrash ", + "wi", + "thout a re", + "-render (#129)", + "\n- Batch **font l", + "oad", + "ing**", + " once ", + "per", + " sessi", + "on (#13", + "0)\n-", + " Pin clipboa", + "rd poli", + "cy in the re", + "lease b", + "uild (#131)\n- Cac", + "he token ", + "refr", + "esh behi", + "nd a f", + "eature", + " flag (#132)", + "\n", + "- P", + "refetc", + "h **list ", + "virtualizati", + "", + "on", + "** behin", + "d a fea", + "tur", + "e flag", + " (#13", + "3", + ")\n-", + " Debounce **gestu", + "re confl", + "ict", + "s*", + "* behind a f", + "eature fl", + "ag ", + "(#134", + ")\n- Cache **font", + " loading** onc", + "e", + " per session (#135", + ")\n- Batch `col", + "d sta", + "rt", + "` withou", + "t a re-", + "render (#136", + ")\n- Cache **col", + "d start** per ", + "navigation en", + "tr", + "y (#13", + "7)", + "\n- Measure", + " push tok", + "ens un", + "", + "der a mem", + "ory cap ", + "(#138)\n- Inline ", + "`dark mode toke", + "n", + "s` with ", + "a", + " 200 ms budget (", + "#139)\n- Debounce ", + "crash reporti", + "ng without ", + "a re-r", + "e", + "nder (#140)\n- Tri", + "m dark mode to", + "kens on", + " the JS th", + "", + "read onl", + "y (#141", + ")\n-", + " Cac", + "he font", + " loadin", + "g per navigation", + " entry (#", + "142)\n- Trim p", + "ush tokens ", + "", + "in the release bu", + "ild (#143)\n- Defe", + "r accessibil", + "ity", + " labels and ", + "log the d", + "el", + "ta (#144)\n- Inline", + " **cold start** ", + "before the firs", + "", + "t fram", + "e (#145", + ")\n- Audit cra", + "sh reporting bef", + "ore the first fra", + "me (#146)\n- Audit", + " text select", + "ion ", + "under a memor", + "y cap (#147)\n", + "-", + " Trim **schema", + " migration** befo", + "re", + " the first", + "", + " frame ", + "(#148)\n- T", + "rim **image c", + "ache** per na", + "vigation entry ", + "(#149", + ")\n- Inline ac", + "c", + "essibility label", + "s beh", + "in", + "d a ", + "featu", + "re", + " flag (#150)\n", + "- T", + "rim teleme", + "try s", + "ampling and log t", + "he delta", + " (#151)\n- Pin d", + "ark mode ", + "tokens behind a fe", + "ature flag (#152)\n", + "- Measure **font", + " loading** on ", + "the ", + "JS thread o", + "nly", + " (#1", + "53)\n- Def", + "er ", + "image cache o", + "n th", + "e JS th", + "read onl", + "y", + " (#", + "154)\n- ", + "Audit dark ", + "mode t", + "okens witho", + "ut a re-render", + " (#15", + "5)\n- Measu", + "re acce", + "ssibil", + "i", + "ty labels ", + "once per sessio", + "n (#156)", + "\n- Debounce acce", + "", + "", + "ssibility la", + "bels with a 200 ", + "ms", + " budget (#157)\n- D", + "eboun", + "ce ", + "bundle size ", + "in the relea", + "se build (#158)", + "\n- Defer", + " gesture c", + "", + "onflicts", + " un", + "der a m", + "emory cap (#159)", + "\n- ", + "Pin token ref", + "resh b", + "ehind", + " a feature fla", + "g (", + "#160)\n- Defer ", + "font lo", + "adi", + "ng with a 200 ms", + " bud", + "get (#161)\n-", + " Prefetch **fon", + "t", + " loading** w", + "ith a ", + "200 ms ", + "budge", + "t (#162", + ")\n", + "- Trim", + " layout thr", + "ash and l", + "og ", + "the de", + "lta (#163)\n- Measu", + "re bundle si", + "ze with a 2", + "00 ms ", + "budget (#16", + "4)\n- Defer `", + "crash reportin", + "g", + "` and log th", + "e delta ", + "(", + "#165)\n- Pin `gestu", + "re ", + "conflicts` with", + "out", + " a re-rend", + "er (#1", + "66)\n- Trim clipb", + "oard policy once ", + "per", + " ", + "session (#16", + "7)\n- Pre", + "fe", + "tch offlin", + "e queue in the rel", + "ease", + " build (#168)\n- ", + "Measure deep l", + "inks in ", + "the rele", + "ase buil", + "d (#1", + "69)\n-", + " Batch **lay", + "out thrash**", + " ", + "per navigation ", + "entry (#17", + "0)\n-", + " Meas", + "", + "ure **l", + "ayout thrash** an", + "d log t", + "he delta (#1", + "7", + "1)\n- Trim offline", + "", + " queue p", + "er navigation", + " ", + "entry (#17", + "2)\n- Cache **la", + "", + "yout thras", + "h** before the ", + "first frame (", + "#173)\n- Batch **a", + "ccess", + "ib", + "ility labe", + "ls** before the f", + "irs", + "t f", + "ram", + "e (#174)\n- Trim", + " dark ", + "mode tokens per na", + "", + "vigation ", + "ent", + "ry (#175)\n- ", + "In", + "line `font loading", + "` once p", + "er session", + " (#1", + "76)\n- Me", + "a", + "sure `sc", + "hema migration` wi", + "thout a re-ren", + "der (#", + "177)\n-", + " Audit **", + "dark mode t", + "okens", + "*", + "* u", + "nder a memory c", + "ap (#178)\n- Pin ", + "token ref", + "resh", + " with", + " a 20", + "0 ms budge", + "t (#179)", + "\n- Measure clipboa", + "rd poli", + "cy on the ", + "JS thread only (", + "#180)\n- De", + "bou", + "nce ge", + "sture confl", + "icts b", + "efore the firs", + "t fr", + "ame ", + "(#181", + ")\n- Pr", + "efetc", + "", + "h", + "", + " retry ba", + "ckoff un", + "der a m", + "emory ca", + "p (#", + "182)\n- Trim ", + "**clipbo", + "ard policy** b", + "ef", + "ore the", + " first frame (#183", + ")\n- Batch d", + "ark mode token", + "s ", + "wit", + "hout a re", + "-re", + "nder (#184)\n- T", + "rim **image cache*", + "* in", + " the", + " release ", + "build (#", + "185)", + "\n- Inline f", + "ont load", + "in", + "g ", + "once p", + "er session (#186)", + "\n- Batch co", + "ld star", + "t per navigation e", + "ntry (#187)\n- Pr", + "efetc", + "", + "h", + " `text sel", + "ection` on", + "ce p", + "er session (#188)", + "\n- Bat", + "ch d", + "eep links befor", + "e t", + "he first fra", + "me (#189)\n", + "- Audit `cold s", + "tart` be", + "hind a feature", + " flag (#190)\n- ", + "Trim", + " list virtual", + "izatio", + "n before the fir", + "st frame (", + "#191", + ")\n- Measure", + "", + " `layout thras", + "h` onc", + "e per ", + "session (#1", + "92)\n- Trim retry", + " back", + "off per navigat", + "ion entry (#19", + "3)\n- Measure text", + " selecti", + "on behin", + "d a feature", + " flag (#194", + ")\n- Batch **i", + "mage ", + "cache** in the ", + "rele", + "ase buil", + "d (#", + "195)\n-", + " Debounce `layout", + " thr", + "ash", + "` per naviga", + "tion entry (#19", + "6)\n- Au", + "dit `offline queu", + "e` with", + " a 200 m", + "s", + " budget (#197", + ")\n- Measure ", + "`schema m", + "igration` on", + " the JS thread ", + "onl", + "y ", + "(#198)\n- Trim `i", + "mage cache` on", + " the JS thread ", + "only ", + "(#199)\n", + "- Pin image ", + "cache o", + "nce per session (", + "#", + "200)\n- Defer", + " text selectio", + "n without a re-", + "render (#201)", + "\n- ", + "Cac", + "he gest", + "ure co", + "nflicts on ", + "the JS thread onl", + "y (#202)\n", + "- Prefetc", + "h gestur", + "e confli", + "", + "cts on t", + "he JS th", + "read only ", + "(#20", + "3)\n- Audit pus", + "h tok", + "ens and log ", + "the d", + "elta (#2", + "04)\n- Batc", + "h **font l", + "oading** and lo", + "g the delta (#2", + "05)\n-", + " Audit **sche", + "ma migrati", + "on** ", + "once per se", + "s", + "sion (#206)\n-", + " Pin sch", + "ema migratio", + "n on th", + "e JS thre", + "ad only (#", + "207)\n- Pin tel", + "emetry sampling be", + "hind a feature fl", + "ag (#208", + ")\n- Batch pu", + "s", + "h tokens o", + "n the ", + "JS t", + "hre", + "ad only (#20", + "9)\n- P", + "in retry backoff u", + "nder a ", + "memory cap ", + "(#210)\n- Audit ", + "clipboard polic", + "y on the JS thread", + " on", + "ly (#211)\n- C", + "ache offline queue", + " per navigat", + "ion ", + "entry (#212)\n-", + " Cache `token refr", + "e", + "sh` under a memory", + " cap (#213)\n-", + " ", + "", + "Defer **layout t", + "hrash** and", + " log the delta", + " (#21", + "4)\n- A", + "udit dark", + " mo", + "de tokens and l", + "og the delt", + "a (#21", + "5)\n- Audit lis", + "t virtualizat", + "ion on t", + "he J", + "S thr", + "ead only (#", + "216)\n- Cache t", + "oken refre", + "sh on the J", + "S ", + "thread only (", + "#217)", + "\n- Batch tele", + "", + "metry sampling onc", + "e per", + " session (#2", + "18)\n- ", + "Audit bundle siz", + "e before the f", + "irst frame", + " ", + "(#219)\n- Measu", + "r", + "e", + " gesture confli", + "ct", + "s with a", + " 200 ms budget ", + "(#220)\n- Mea", + "sure sch", + "", + "em", + "a migration o", + "nce per sessio", + "n (#221)\n", + "- Audit crash repo", + "rting in the relea", + "se build (#222)\n- ", + "Inline bu", + "ndle size bef", + "ore the ", + "first frame (#223)", + "\n- Debounce image", + " cache wit", + "h a 20", + "0 ms b", + "udget (#224)\n- B", + "atch bundle ", + "size", + " o", + "nce per session", + " (#225)\n-", + " Inline gestur", + "e conflicts wit", + "hout a re-rende", + "r (#22", + "6)\n-", + " Cache telemetr", + "y sam", + "pl", + "ing befor", + "e ", + "the first", + " frame (#227)\n-", + " Measure telemetry", + "", + " sampling wi", + "thout a r", + "e-", + "render (#228)\n- ", + "Defer t", + "ext selection pe", + "r navig", + "ation entry (", + "#229)\n- D", + "ebou", + "nce retry backof", + "f behi", + "nd a feature", + "", + " flag ", + "(#230)\n- Defer ", + "**push tokens**", + " be", + "fore t", + "he fir", + "st frame (#231)", + "\n- Cache push", + " toke", + "ns under a ", + "me", + "mory c", + "ap (#2", + "32)", + "\n- Pin cold ", + "start under a memo", + "r", + "y cap (#233)\n- Bat", + "ch telemetry sa", + "mpling on the ", + "JS thread o", + "nly (#234)", + "\n-", + " Pin **token ref", + "resh** b", + "efor", + "e the first fra", + "me (#23", + "5)\n- Batch **t", + "e", + "leme", + "try samp", + "ling** in the", + " release buil", + "d (#236)\n- Pre", + "fetch crash repo", + "rtin", + "g per ", + "navi", + "gation", + " entry (#237)\n", + "- Inline `off", + "line queue` with", + " a 2", + "00 ms budge", + "t (#238)\n- ", + "Defer dee", + "p li", + "nks a", + "n", + "d log the delta", + " (#239", + ")\n- Audit **", + "layout t", + "hrash** ", + "on the JS", + " thread on", + "ly (", + "#240)\n- Defer toke", + "n refresh on the J", + "S", + " thread only (#2", + "41)\n- Prefetch **", + "f", + "ont loadin", + "g**", + " pe", + "r", + " navigation ent", + "ry (#242)", + "\n- Batch **co", + "ld", + " start** pe", + "r navigat", + "ion entry ", + "(#243)\n- ", + "Audit ", + "**sche", + "ma migratio", + "n", + "** and log th", + "e delta ", + "(#24", + "4)\n- D", + "efer offline q", + "ueue before", + " th", + "e", + " ", + "first frame (#", + "245)\n- I", + "nline **gesture c", + "onflicts** before ", + "the f", + "irst frame (#24", + "6", + ")\n- ", + "Cach", + "e dark mode to", + "kens in the", + " release ", + "bu", + "ild (#247", + ")", + "\n- Inline **retr", + "y b", + "ackoff** without a", + " re-r", + "ender (#248", + ")\n- D", + "efer font lo", + "", + "ading un", + "der a memory cap ", + "(#249)\n- Batch dee", + "p links and log th", + "e ", + "delta (#250)", + "\n- P", + "in ", + "sch", + "em", + "a migration once p", + "er ", + "s", + "ession (#251", + ")\n- Audit ", + "locale fallbac", + "ks behind a ", + "feature fla", + "g ", + "(#252)\n- Inlin", + "e **cold ", + "start**", + " once pe", + "r sessio", + "n", + " (#253)\n- ", + "Batc", + "h", + " **clipboa", + "rd policy** pe", + "r n", + "a", + "vigati", + "on e", + "ntry (#25", + "4)\n- Tri", + "m i", + "mage ", + "cache behind a", + " featur", + "e flag (#25", + "5)\n- Debounce ac", + "c", + "essibility l", + "ab", + "els befor", + "e the first ", + "frame (", + "#256)\n- Pin", + " accessibility", + " lab", + "els behind a f", + "eatu", + "re flag (#", + "257)\n- Cache `lo", + "cale fall", + "backs` under", + " a mem", + "ory cap (#258)\n-", + " Deboun", + "ce **schema migr", + "ation** before th", + "e first frame (#25", + "9)\n", + "- ", + "Debounc", + "e **bundle siz", + "e*", + "* with a ", + "200 ", + "ms budget (#260)\n-", + " Audit `bun", + "dle size` ", + "with a", + " 200 ms budget (#", + "261)\n- Mea", + "sure retry ", + "backoff on", + "ce per", + " session (", + "#262)", + "\n- Ba", + "t", + "ch **", + "retry bac", + "koff** on", + "ce per ses", + "sion (#263)\n-", + " Measur", + "e schema migrati", + "on wi", + "t", + "h a 200 m", + "", + "s budget (#264", + ")\n- In", + "line `dar", + "k mod", + "e tokens`", + " und", + "er a ", + "me", + "mory cap (#", + "265)\n- Cache", + " ", + "font loading beh", + "ind a feature flag", + " (#266)\n- Me", + "asure offline", + " queue onc", + "e per ", + "sess", + "ion", + " (#267)\n- Pref", + "etc", + "h deep", + " links under a mem", + "ory cap (#2", + "68)\n- P", + "re", + "fetch font loading", + " wit", + "hout a re-render", + " (#269)\n- Def", + "er `crash ", + "repor", + "ting` once", + " per se", + "ssion (#270)\n-", + " Measure token", + " refresh withou", + "t ", + "a re-rend", + "er (#27", + "1)\n- Cac", + "he layout ", + "thrash and log", + " the delta (#272)", + "\n- Inline **font l", + "oad", + "ing** in", + " the rele", + "ase build ", + "(#27", + "3)\n- Audit `text", + " selection` b", + "efore the fi", + "rst", + " frame (#274)", + "\n- ", + "Pin", + " `list virtual", + "ization`", + " and log the de", + "", + "lta (#275)\n-", + " Defer clipboa", + "rd policy i", + "n the release bui", + "", + "", + "ld (#276)\n- Pref", + "et", + "ch `list virtual", + "ization", + "` before the fir", + "st frame (", + "#277)\n- Bat", + "ch", + " `loc", + "ale fallbacks` ", + "without", + " a ", + "re-render (#278)\n-", + " Cache token refre", + "sh behind a", + " feat", + "ure flag ", + "(#279)\n-", + " Debounce list vir", + "tualiza", + "t", + "ion on ", + "the ", + "JS thread only (#2", + "80)\n- Defer layout", + " thrash u", + "nder a memory cap", + " (#281)\n- Me", + "asure retry back", + "", + "off befor", + "e the first frame ", + "(#282)\n- M", + "e", + "asur", + "e token ref", + "r", + "esh per navigatio", + "n", + " ", + "", + "entry (#283)\n-", + " Inline schema m", + "", + "igration behi", + "nd ", + "a feature flag", + " (#284)\n- Meas", + "ure `dark mode tok", + "ens` p", + "er n", + "avi", + "gation en", + "t", + "ry (#285)\n- B", + "atch re", + "try backoff", + " be", + "fore the first fr", + "ame ", + "(#286)\n- Pin ret", + "ry back", + "off bef", + "ore the", + " first f", + "rame (#28", + "7)\n- Defer imag", + "e cache per navi", + "gation en", + "t", + "ry (#288)\n- Audit", + " telemetry samplin", + "g with a", + " ", + "200 ms budget (#2", + "89)\n- Inlin", + "e `acces", + "sibility ", + "labels`", + " under a", + " memory cap (#2", + "90)\n- Batc", + "h text selec", + "tion in the rel", + "ea", + "se", + " buil", + "d (#291", + ")\n- Trim telemet", + "ry sampling i", + "n th", + "e release ", + "", + "build (#292)\n- D", + "efe", + "", + "r **", + "push ", + "tokens** without ", + "a re-rend", + "er (", + "#", + "2", + "93)\n- Pin `retry", + " backoff` ", + "under a memory ", + "cap (#294", + ")", + "\n- Trim acces", + "sibilit", + "y ", + "labe", + "ls an", + "d log the", + " delta (#295)\n- M", + "easure deep links ", + "before the ", + "fir", + "st frame", + " (#296)\n- M", + "", + "easure *", + "*token refresh", + "** once per ses", + "sion", + " ", + "(", + "#297", + ")\n-", + " Debo", + "unce offline queu", + "e withou", + "t a", + " re-render", + " (#2", + "98)\n-", + " Batch crash", + " reporting and lo", + "g the de", + "lta (#", + "299)\n- Deboun", + "ce loc", + "ale fallba", + "cks", + " ", + "w", + "i", + "th a 200 ms bud", + "", + "get (#300", + ")\n- Audit", + "", + " image cache und", + "er", + "", + " a memory ca", + "p (#301)\n- Defe", + "", + "r layout thrash wi", + "th a 200 ms budget", + " (#302)\n", + "- Deboun", + "ce **font loa", + "din", + "g", + "** in th", + "e r", + "e", + "lease bu", + "ild (#303)\n", + "- Trim **bund", + "le si", + "ze** once p", + "er se", + "ssio", + "", + "n (#304)\n- T", + "rim token", + " refresh before th", + "e first frame (#3", + "05)\n- Prefet", + "ch schema migrat", + "ion", + " behind a feature", + " flag ", + "(#", + "306)\n- Pref", + "etch dark ", + "mode tokens withou", + "t a re-render ", + "(#307", + ")", + "\n", + "- Trim clip", + "", + "board poli", + "cy per navigation", + " entry ", + "(#308)\n- Deb", + "oun", + "ce da", + "rk mode tokens bef", + "ore the first fram", + "e (#309)\n- Audit `", + "gesture confl", + "icts` with", + "out a ", + "re-", + "render (#3", + "10)\n-", + " Measure retry bac", + "koff in the r", + "elea", + "se build (#311", + ")\n- Prefetch font", + " loading ", + "on the JS thread ", + "only (#", + "31", + "2)\n- Bat", + "ch bun", + "dle size ", + "in t", + "he releas", + "e build ", + "(#313)\n- Ba", + "tch te", + "lemetry sa", + "mpling on th", + "e JS t", + "hread only (#314", + ")\n- Audit **schem", + "a ", + "migration** wi", + "th a 200 ms budg", + "et ", + "(#315)\n- Deb", + "ounce font lo", + "ading with", + "out a re-render (", + "#316", + "", + ")\n- Pin **gestu", + "re con", + "flicts** on th", + "e JS thread only", + " (#317)", + "\n- ", + "Trim ret", + "ry bac", + "ko", + "f", + "f pe", + "r navigat", + "ion entry", + " (#318)\n", + "- Inline offli", + "ne queue a", + "nd log", + " the ", + "delta (#319)\n- ", + "Def", + "er `layout thrash", + "` before ", + "the ", + "first frame (#320", + "", + ")\n- Deboun", + "ce gesture con", + "flict", + "s per navig", + "ation e", + "n", + "try", + " (#321)\n-", + " ", + "Audit layou", + "t thrash", + " in the re", + "lease build ", + "", + "(#322)\n- Pin **cra", + "sh rep", + "orting** on the JS", + " ", + "thre", + "ad on", + "ly (#323)\n- Defer ", + "", + "**locale fallb", + "ack", + "s** b", + "efore the", + " ", + "fir", + "st f", + "rame (#324)\n", + "- Tri", + "m", + " bundle si", + "ze before ", + "the f", + "irst frame (#32", + "5", + ")\n- D", + "", + "ebounce `offli", + "n", + "e queue` with", + "out a re-rend", + "er (#326)", + "\n- Audit **cold ", + "start** on t", + "he JS ", + "thread only (#32", + "7)\n-", + " Audit token refr", + "es", + "h once per ", + "sessi", + "on (#328)\n- Audit ", + "`gesture conflic", + "ts` ", + "once p", + "er session (#32", + "9)\n- Trim **cold ", + "start**", + " in the release b", + "uil", + "d (#330)\n- Pin b", + "undl", + "e si", + "z", + "e per navigation e", + "ntry (", + "#331)\n- Mea", + "sure `a", + "c", + "cessibility l", + "abels` once", + " per", + " se", + "ssion (#332)\n", + "- Measu", + "re cli", + "pboard policy", + " under a memory ca", + "p (#333)\n- Audi", + "t bundle si", + "ze with", + "out a re-render ", + "(#", + "334)\n- Mea", + "s", + "ure", + " gesture co", + "nf", + "licts once p", + "er session ", + "(#335)", + "\n- Batch push tok", + "ens behind a f", + "", + "eature flag (#336", + ")\n- Cache `bundl", + "e size`", + " under a m", + "emory c", + "ap (#337)\n- T", + "rim **local", + "e fa", + "llback", + "s** per na", + "vigation entry ", + "(#338)\n- ", + "Debounce `pus", + "h ", + "tokens` per nav", + "ig", + "ation entry", + " (#339)\n- Mea", + "sure clipbo", + "", + "a", + "rd policy on the J", + "S thread onl", + "y ", + "(#3", + "40)\n- Inline gest", + "", + "ure ", + "conflicts with ", + "a 200 m", + "s budget (#341)", + "\n- Prefetch layout", + " t", + "hrash under a memo", + "r", + "y cap (#342)\n- ", + "P", + "refetch **da", + "rk mod", + "e tok", + "ens** wi", + "th", + " a 200 ms budget (", + "#343)\n- Def", + "er acc", + "essi", + "bility labels in ", + "the release b", + "u", + "ild ", + "(#344)\n", + "- Inline", + " cold start wit", + "hout a re-r", + "ender (#345", + ")\n-", + " Defer retry backo", + "ff per navigation", + " entr", + "y (#346)\n", + "- Cache ges", + "ture conf", + "licts and log ", + "the delta (#347", + ")\n- Defer clip", + "board p", + "olicy on ", + "the JS thread", + " only (#3", + "48)\n- Prefetch ", + "", + "accessib", + "ility labels wi", + "th", + "out a re-ren", + "der (#349)\n- Mea", + "sure", + " **text selecti", + "on", + "** p", + "er navigati", + "on e", + "ntry (#350)", + "\n- Bat", + "ch **offline qu", + "eue** with a 2", + "00 ms budget (#3", + "51)\n-", + " Measure tok", + "en refresh behind ", + "a feature flag ", + "(#", + "352)\n- Cache **c", + "", + "li", + "pboard p", + "o", + "licy** ", + "and log", + " the delta (#35", + "3)\n- Batch pu", + "sh tokens", + " behind a fe", + "a", + "", + "ture flag (#35", + "4)\n- Pin **gest", + "", + "ure conflicts", + "** in the rel", + "ease bui", + "ld (#3", + "55)\n-", + " Cache local", + "e ", + "fallbacks u", + "", + "nder a memory", + "", + " cap (#35", + "6)\n- Inline `cold ", + "start` on th", + "e JS", + " ", + "thread only (#3", + "57)\n- Audit text", + " sele", + "ction with", + " a 200 ms budget (", + "", + "#358)\n- T", + "", + "r", + "im ", + "schema mi", + "gration und", + "er a memo", + "ry c", + "ap (#359)", + "\n- Audit cr", + "ash reporting", + " w", + "itho", + "ut ", + "a re-render (#", + "360)\n- Aud", + "it **crash", + " reporti", + "ng**", + " on the JS t", + "hr", + "ead only (#361)\n", + "- Measu", + "", + "re `p", + "us", + "h", + " tokens` without a", + " re-render", + " (#362)\n- Batch", + " accessibili", + "ty la", + "bels before the fi", + "", + "rst frame (#363)\n-", + " Pre", + "fetch **telemet", + "r", + "y sampling** pe", + "r navigati", + "", + "on entry (", + "#364)\n- Trim cold", + " st", + "art without a r", + "e-render (#365)\n-", + " Pr", + "efetch push to", + "kens on the ", + "JS thread only", + " (#366)\n-", + " Pin `telemet", + "ry ", + "sampling`", + " once per sess", + "ion (#", + "367)", + "\n- Cache", + " `retry backoff", + "` behind", + " a f", + "eature flag (#368", + ")\n- Batch `", + "cold start` once p", + "er sess", + "ion (#369)", + "\n- I", + "n", + "line pu", + "sh tokens on t", + "he JS thread ", + "only (", + "#370)\n-", + " Pin schem", + "a m", + "igra", + "tion before the fi", + "rst frame (#371)\n", + "- Batch la", + "yout thrash wit", + "h ", + "a 2", + "", + "00 ", + "ms", + " budget (#3", + "72)\n- Pin **col", + "d star", + "t** in the relea", + "se build ", + "(#373)\n- Pin ", + "layout thrash with", + "out a re-rende", + "r (#374)", + "\n- Defer `crash r", + "eporting` without", + " a re-ren", + "der ", + "(#375)\n- Defer ", + "list", + " virtua", + "lization b", + "ehind a fe", + "ature f", + "lag (#376)\n- Ca", + "che image cache ", + "under a mem", + "ory ca", + "p (#377)", + "\n- Prefetch offl", + "ine queue w", + "ithout a re-rend", + "e", + "r (#", + "378)\n- Prefetch ", + "bu", + "ndle size per navi", + "gatio", + "n entry (#379)\n- ", + "P", + "refetch telemetry ", + "s", + "ampl", + "ing without a re", + "-render (#", + "380)\n- Prefe", + "tch retry b", + "a", + "cko", + "ff once", + " per session (#", + "381)\n- Me", + "a", + "sur", + "e clipboard ", + "policy and ", + "log the de", + "lta (#382)\n- ", + "", + "Audit gestur", + "e confli", + "cts", + "", + " per nav", + "igation entry (#38", + "3)\n- Trim o", + "ffline queue per n", + "avigat", + "", + "ion entry (#38", + "4)\n- Audit *", + "*loca", + "le", + " fallba", + "c", + "ks** on the JS ", + "thread ", + "only (#385)\n- ", + "Trim c", + "lipboard", + " policy and", + " log the delta (#", + "386)\n- Me", + "asu", + "re d", + "ark mode tokens ", + "and log the de", + "lta (#387)\n", + "- Defer token ", + "refresh without a", + " re-re", + "nder (#388)", + "\n- Trim *", + "*cold sta", + "rt** in the", + " release build (#", + "389)\n- Measure `", + "local", + "e fallbacks", + "`", + " without a re", + "-rend", + "er (#390)\n- De", + "f", + "er `tok", + "en refres", + "h` on ", + "the JS th", + "read o", + "nly (#391)\n-", + " Measure ", + "gestu", + "re conflicts and", + " l", + "og the", + " ", + "delta (#392)", + "\n- Deb", + "ounce d", + "ark", + " mode token", + "s in the relea", + "se build (", + "#393)\n-", + " Measure gesture", + " conflicts in the", + " rel", + "ease ", + "build (#3", + "94)\n- Prefe", + "t", + "ch list vir", + "tua", + "lization be", + "fo", + "re ", + "the first ", + "frame (#395)\n- D", + "efer layout thrash", + " and log the", + " delt", + "", + "a (#", + "396)\n- Pref", + "etch **font lo", + "ading** on", + " ", + "the JS thread onl", + "y (#397)\n", + "- ", + "B", + "atch `offline ", + "qu", + "e", + "ue` in the releas", + "e", + " build (#", + "398)\n- Pin gestu", + "re", + " conflicts wit", + "h a", + " 200 ms budget (#", + "399)\n", + "- Prefetch bundle", + " size without ", + "a re-render", + " (", + "#400)", + "\n- Debounce ge", + "sture con", + "flicts on the J", + "S th", + "read only (#401", + ")\n- Audit s", + "che", + "ma migration be", + "fore th", + "e", + " firs", + "t frame (#402)", + "\n- Inli", + "ne d", + "eep ", + "links on ", + "the", + " JS", + " t", + "hread", + " on", + "ly (#403)", + "\n- Audit", + " gesture c", + "onflicts ", + "with a 200", + " ms budge", + "t (#404)\n- ", + "Batch cold sta", + "rt per navigat", + "ion entry (#405", + ")\n- Defe", + "r `cold start` wit", + "h a 2", + "00 m", + "s ", + "budget (#406)\n-", + " Me", + "asure", + " c", + "lipboard policy be", + "hind a feat", + "ure flag (#40", + "7)\n- Prefet", + "ch retry backof", + "f in t", + "he release", + " build (#", + "408)\n- Trim crash", + " reporting", + " behind a feature ", + "flag (#409", + ")", + "\n- Cache **", + "gesture conflicts*", + "* per", + " nav", + "igation ent", + "ry (#410)\n- Inline", + " gestu", + "r", + "e conflic", + "ts behind a featu", + "re flag (", + "#", + "411)", + "\n- Inline telemetr", + "y samplin", + "g behind ", + "a feature flag (", + "#412)\n- D", + "ebounce text s", + "electi", + "on on the", + " JS", + " thre", + "ad only ", + "(#413)\n- Inlin", + "e list virtual", + "izatio", + "n per navigation", + " entry (#414)\n-", + " Batc", + "h telemetry sam", + "pling and l", + "og the delta (#415", + ")\n- Measure d", + "ark ", + "mode to", + "kens", + " in the release", + " bui", + "ld (#416", + ")\n- Trim font loa", + "ding on", + " the JS thread o", + "nly (", + "#", + "417)\n- De", + "fer gesture ", + "conflicts in the", + " ", + "release build (#41", + "8)\n- Pref", + "etch", + " crash rep", + "orting a", + "nd log the del", + "ta (#419)", + "\n- Pin offli", + "ne queu", + "e with a 20", + "0 ms bu", + "d", + "get (", + "#420)\n" + ] +} diff --git a/conformance/selection/incremental-projection.test.ts b/conformance/selection/incremental-projection.test.ts new file mode 100644 index 0000000..0116f1a --- /dev/null +++ b/conformance/selection/incremental-projection.test.ts @@ -0,0 +1,293 @@ +/** + * Incremental projection, held over the corpus and over a real stream. + * + * WHY THIS FILE EXISTS. A settled prose run GROWS: `segmentRuns` merges every + * adjacent settled flowing block into one run, so an ordinary answer is one run + * that gains a block on every settle. Reprojecting it each time is O(document) + * per settle and O(document²) over a message — measured at 668,995 characters + * projected for a 14 kB document and 2,528,198 for a 28 kB one. `projectRun` + * therefore takes a `previous` projection and extends it, and + * `createRunProjectionCache` is what holds one per run. + * + * That optimisation is only safe if an extended projection is INDISTINGUISHABLE + * from a fresh one, and "indistinguishable" is a strong claim: the projector + * merges a chunk into the preceding piece when the two are linear in the + * source, records marks as constructs close and sorts them at the end, numbers + * embeds by position, and refuses to grow an embed's piece. All four are state + * carried across a block boundary. Hand-built fixtures check the constructs + * somebody thought of (src/selection/__tests__/mapSelection.test.ts, "projectRun + * (incremental)"); this checks every construct in the CommonMark suite and every + * shipped fixture, parsed by the engine the app actually runs. + * + * The second half measures rather than compares: it replays fixtures through a + * real `StreamSession` and gates the amplification — projected characters over + * document characters — at a constant, which is the invariant docs/ + * PERFORMANCE.md and docs/BENCHMARKS.md quote. bench/projection.mjs reports the + * same number with the transcripts, in the open. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import type { ParsedDocument } from '../../src/document/nodes'; +import { parseDocument } from '../../src/engine/Engine'; +import { + describeNative, + linkNativeEngineAsDefault, +} from '../../src/engine/native/__tests__/support'; +import { presets } from '../../src/engine/options'; +import type { EngineOptions } from '../../src/engine/options'; +import { projectRun } from '../../src/selection/mapSelection'; +import { segmentRuns } from '../../src/selection/runs'; +import type { EmbedLookup, RunSegment } from '../../src/selection/runs'; +import { StreamSession } from '../../src/stream/StreamSession'; +import { createRunProjectionCache } from '../../src/view/projectionCache'; +import type { RunProjectionCache } from '../../src/view/projectionCache'; +import { runKey } from '../../src/view/runIdentity'; + +const FIXTURE_DIR = path.resolve(__dirname, '..', 'fixtures'); +const SPEC_PATH = path.resolve(__dirname, '..', 'vendor', 'spec.json'); + +interface Case { + readonly label: string; + readonly source: string; +} + +function loadCorpus(): Case[] { + const cases: Case[] = []; + const spec = JSON.parse(fs.readFileSync(SPEC_PATH, 'utf8')) as { + markdown: string; + example: number; + }[]; + for (const entry of spec) { + cases.push({ label: `spec example ${entry.example}`, source: entry.markdown }); + } + for (const file of fs.readdirSync(FIXTURE_DIR).filter((f) => f.endsWith('.md'))) { + cases.push({ + label: `fixture ${file}`, + source: fs.readFileSync(path.join(FIXTURE_DIR, file), 'utf8'), + }); + } + return cases; +} + +const corpus = loadCorpus(); + +linkNativeEngineAsDefault(); + +/** `run` restricted to its first `count` blocks. */ +function prefixRun(run: RunSegment, count: number): RunSegment { + const blocks = run.blocks.slice(0, count); + return { + ...run, + blocks, + span: { + start: blocks[0].span.start, + end: blocks[blocks.length - 1].span.end, + }, + }; +} + +/** + * Grows `run` one block at a time through a cache, asserting after every step + * that the result deep-equals the projection from scratch. Returns the number + * of steps taken, so a caller can prove the corpus actually exercised runs with + * more than one block in them. + */ +function growAndCompare( + run: RunSegment, + doc: ParsedDocument, + options?: { embed?: EmbedLookup }, +): number { + const cache = createRunProjectionCache(); + for (let count = 1; count <= run.blocks.length; count += 1) { + const grown = prefixRun(run, count); + expect(cache.project(grown, doc, options)).toEqual( + projectRun(grown, doc, options), + ); + } + return run.blocks.length; +} + +describeNative.each([ + ['llmChat', presets.llmChat], + ['everything', presets.everything], +] as [string, EngineOptions][])('incremental projection (%s)', (_name, options) => { + it('grows every corpus run to exactly the full projection', () => { + let steps = 0; + for (const { label, source } of corpus) { + const doc = parseDocument(source, options); + for (const run of segmentRuns(doc)) { + if (run.standalone) { + continue; + } + try { + steps += growAndCompare(run, doc); + } catch (err) { + throw new Error(`${label}: ${(err as Error).message}`); + } + } + } + // The property is vacuous on one-block runs, so make the corpus prove it + // supplied plenty of multi-block ones. + expect(steps).toBeGreaterThan(500); + }); + + it('grows identically when an embed lookup is claiming nodes', () => { + // Links and code blocks: one inline, one block, so the seam between two + // projected blocks is crossed with an embed on either side of it. + const embed: EmbedLookup = (node) => + node.kind === 'link' || node.kind === 'codeBlock' + ? { width: 100, height: 40, text: '[card]' } + : undefined; + let claimed = 0; + for (const { label, source } of corpus) { + const doc = parseDocument(source, options); + for (const run of segmentRuns(doc, { embed })) { + if (run.standalone) { + continue; + } + try { + growAndCompare(run, doc, { embed }); + } catch (err) { + throw new Error(`${label}: ${(err as Error).message}`); + } + claimed += projectRun(run, doc, { embed }).embeds?.length ?? 0; + } + } + expect(claimed).toBeGreaterThan(50); + }); +}); + +/** + * The measuring instrument: a pure `EmbedLookup` that claims nothing and sums + * the source extent of the top-level nodes it is offered. The projector offers + * a run's own blocks with `topLevel: true` and every descendant with false, so + * that sum is exactly "source characters projected" — the number the audit + * measured. Claiming nothing leaves the projection byte-identical to one with + * no lookup at all (asserted by the corpus case above, which projects both + * ways). + */ +function meter(): { + embed: EmbedLookup; + measure: (body: () => T) => T; + chars: () => number; +} { + let counting = false; + let chars = 0; + return { + embed: (node, context) => { + if (counting && context.topLevel) { + chars += node.span.end - node.span.start; + } + return undefined; + }, + measure: (body) => { + counting = true; + try { + return body(); + } finally { + counting = false; + } + }, + chars: () => chars, + }; +} + +interface Replay { + /** Source characters handed to the projector across the whole stream. */ + projected: number; + /** The finished document's length. */ + source: number; + /** The largest single projection — the number an old late settle blew up. */ + worst: number; +} + +/** + * Streams `source` in `chunk`-character deltas through a real `StreamSession` + * and runs the view pipeline on every commit: `segmentRuns`, one cache per + * `runKey` (which is what `SelectableMarkdown` files its `RunView`s under), and + * `cache.project` per prose run. + */ +function replay(source: string, chunk: number, options: EngineOptions): Replay { + const gauge = meter(); + const caches = new Map(); + const session = new StreamSession({ options }); + let worst = 0; + + const draw = (): void => { + const snapshot = session.snapshot(); + const doc = snapshot.document; + const runs = segmentRuns(doc, { + settledUntil: snapshot.settledUntil, + embed: gauge.embed, + }); + runs.forEach((run, index) => { + if (run.standalone) { + return; + } + const unsettledTail = + snapshot.phase === 'streaming' && run.span.end > snapshot.settledUntil; + const key = runKey(run, index, runs.length, unsettledTail); + let cache = caches.get(key); + if (cache === undefined) { + cache = createRunProjectionCache(); + caches.set(key, cache); + } + const before = gauge.chars(); + gauge.measure(() => cache.project(run, doc, { embed: gauge.embed })); + worst = Math.max(worst, gauge.chars() - before); + }); + }; + + for (let at = 0; at < source.length; at += chunk) { + session.append(source.slice(at, at + chunk)); + draw(); + } + session.finalize(); + draw(); + + return { projected: gauge.chars(), source: source.length, worst }; +} + +/** The longest shipped fixture, repeated to make a document of a given size. */ +function transcript(minimumLength: number): string { + const parts = fs + .readdirSync(FIXTURE_DIR) + .filter((f) => f.endsWith('.md')) + .map((f) => fs.readFileSync(path.join(FIXTURE_DIR, f), 'utf8')); + let out = ''; + while (out.length < minimumLength) { + for (const part of parts) { + out += `${part.trim()}\n\n`; + if (out.length >= minimumLength) break; + } + } + return out; +} + +describeNative('projected-character amplification', () => { + const CHUNK = 18; + + it('stays flat as the streamed document doubles', () => { + const small = replay(transcript(7000), CHUNK, presets.llmChat); + const large = replay(transcript(14000), CHUNK, presets.llmChat); + + const smallAmp = small.projected / small.source; + const largeAmp = large.projected / large.source; + + // Before the cache this ratio grew with the document (47.7× at 14 kB, + // 90.1× at 28 kB). It is now set by how long a block spends as the live + // tail, which is a property of the delta size, not of the document. + expect(largeAmp).toBeLessThan(smallAmp * 1.35); + expect(largeAmp).toBeLessThan(40); + }); + + it('bounds the largest single projection well below the document', () => { + const doc = transcript(14000); + const { worst, source } = replay(doc, CHUNK, presets.llmChat); + + // The number the old design could not bound at all: with a whole message + // in one growing run, the last settle reprojected the whole message. + expect(worst).toBeLessThan(source / 4); + }); +}); diff --git a/conformance/selection/projection-oracle.test.ts b/conformance/selection/projection-oracle.test.ts index 4a43a3f..dd55ea0 100644 --- a/conformance/selection/projection-oracle.test.ts +++ b/conformance/selection/projection-oracle.test.ts @@ -96,6 +96,36 @@ function proseRuns(doc: ParsedDocument): { run: RunSegment; projected: Projected .map((run) => ({ run, projected: projectRun(run, doc) })); } +/** + * Every one-for-one character substitution the projection is allowed to make + * inside a LINEAR piece, keyed by the source character. Anything else that + * changes a character changes its length too (`--` → `–`, `&` → `&`) and + * so cannot appear in a linear piece at all. + */ +const PROJECTED_SUBSTITUTIONS: ReadonlyMap = new Map([ + // A soft break renders as a space — docs/FABRIC-PLAN.md §6.1(a), and the + // guard below this block. + ['\n', ' '], + ['\r', ' '], + // Smart punctuation, under the presets that enable it. Which curly quote + // depends on what preceded it, so both are allowed here. + ['"', '“”'], + ["'", '‘’'], + // A NUL is replaced at decode time (`appendText`, decode.ts). + ['\u0000', '\ufffd'], + // An indented code block's leftover indent. md4c consumes four columns of + // indent to make the block and emits the remainder as SPACES + // (md4c.c:5355-5357, `indent_chunk_str`), so `- foo\n\n\t\tbar` shows two + // spaces for two source tabs — eight columns, six eaten by the item indent + // plus the code indent, two left. The expansion is one-for-one only when + // the leftover column count happens to equal the source character count; + // every other ratio changes the length and so cannot be a linear piece at + // all. Where it IS one-for-one the mapping is honest — display offset i is + // source offset i, both inside the indent — which is exactly the property + // this list exists to enumerate. + ['\t', ' '], +]); + function selectionOffsets(length: number): number[] { if (length <= EXHAUSTIVE_LENGTH_LIMIT) { return Array.from({ length: length + 1 }, (_, i) => i); @@ -106,6 +136,97 @@ function selectionOffsets(length: number): number[] { return offsets; } +/** + * Letters and digits, everything else dropped. + * + * The sub-block sweep compares what was on screen against what the copy shows, + * and the two legitimately disagree about punctuation and whitespace: a soft + * break is a space on screen and a newline in the slice, a smart quote is one + * character on screen and two in the source, a list the selection covered + * whole comes back with markers the screen drew as glyphs. None of that is + * loss. Losing a LETTER is. + */ +function lettersAndDigits(text: string): string { + return text.replace(/[^\p{L}\p{N}]+/gu, ''); +} + +/** Whether every character of `needle` appears in `haystack`, in order. */ +function isSubsequence(needle: string, haystack: string): boolean { + let i = 0; + for (let j = 0; j < haystack.length && i < needle.length; j += 1) { + if (haystack[j] === needle[i]) i += 1; + } + return i === needle.length; +} + +/** + * The part of a display slice that came from the SOURCE — the selection minus + * every synthetic glyph in it. + * + * A marker glyph (`• `, `1. `) is projected from no source at all + * (`piece.source === null`), so a selection that sweeps one and stops short of + * the construct it belongs to copies characters the glyph is not among. That + * is the projection working as designed, not the copy losing text, so the + * glyphs are removed from the side being compared rather than excused + * afterwards. + */ +function sourceBackedSlice( + projected: ProjectedRun, + start: number, + end: number, +): string { + let shown = ''; + for (const piece of projected.pieces) { + if (piece.source === null) continue; + const from = Math.max(piece.textStart, start); + const to = Math.min(piece.textEnd, end); + if (to > from) shown += projected.text.slice(from, to); + } + return shown; +} + +/** + * Source ranges whose text is shown VERBATIM: code spans, code blocks and raw + * HTML. + * + * Inside one of these the screen shows the characters as they are written, so + * `ö` is six characters on screen. Copy hands back that slice, and + * `buildCopyPayload` re-parses it — as markdown, where `ö` is one + * character. The copy did not lose the text; the text stopped being verbatim + * the moment it left the block it was written in. That is exactly the "a slice + * can mean something else standing alone" caveat in `copy.ts`, and it is why + * the sweep skips selections that touch one. + */ +function verbatimRanges(doc: ParsedDocument): { start: number; end: number }[] { + const ranges: { start: number; end: number }[] = []; + visit(doc, (node) => { + if ( + node.kind === 'codeSpan' || + node.kind === 'codeBlock' || + node.kind === 'htmlSpan' || + node.kind === 'htmlBlock' + ) { + ranges.push({ start: node.span.start, end: node.span.end }); + } + return undefined; + }); + return ranges; +} + +/** + * A line that OPENS a block whose visible output depends on text the slice + * does not carry: an HTML block, a code fence, or a link reference definition. + * + * All three render as nothing (or as their own contents, which the slice cut + * off) when the slice is parsed on its own, however ordinary the characters + * looked on screen. `[bar]: /baz` is body text in the middle of a paragraph + * and a definition at the start of one; ```` ```foo ```` is an unterminated + * fence whose body is empty. Same family as `verbatimRanges` — the slice means + * something else standing alone — but detectable only in the SLICE, since in + * the document these lines were nothing of the kind. + */ +const OPENS_AN_UNFINISHABLE_BLOCK = /^ {0,3}(?:<|`{3,}|~{3,}|\[[^\]\n]*\]:)/m; + describeNative.each([ ['llmChat', presets.llmChat], ['everything', presets.everything], @@ -138,6 +259,45 @@ describeNative.each([ expect(runs).toBeGreaterThan(400); }); + /** + * THE INVARIANT `segmentRuns` EXISTS TO ENFORCE, held over the corpus + * instead of over a handful of hand-built blank documents. + * + * A run that projects nothing draws nothing. `resolveRunAttributes` returns + * no attributed string for empty text, both native hosts measure it to a + * 0×0 box, and both skip decoration drawing at zero length — so a FLOWING + * run with no characters is a hole where a block should be. Segmentation + * demotes any group whose blocks all project nothing to standalone runs, + * where the built-in renderers draw the rule, the code box or the quote bar + * instead. + * + * The corpus has 18 such groups under `llmChat` and they are not exotic: + * `## ` (example 79), an unterminated ``` fence (126), `>` (239), a link + * with empty text (484). The first two are what a stream looks like one + * chunk before its content arrives. + * + * This is also the guard on `emitsOwnText` in runs.ts, which mirrors the + * projector's emission rules by hand: a kind that stops projecting text but + * stays on that list starts failing here. + */ + it('every flowing run projects non-empty text', () => { + let runs = 0; + for (const { label, source } of corpus) { + const doc = parseDocument(source, options); + for (const { run, projected } of proseRuns(doc)) { + runs++; + if (projected.text.length === 0) { + const kinds = run.blocks.map((block) => block.kind).join(', '); + throw new Error( + `${label}: a flowing run over [${kinds}] projects the empty ` + + 'string, so it would draw nothing at all', + ); + } + } + } + expect(runs).toBeGreaterThan(400); + }); + it('every mapped selection is an in-bounds, ordered source span', () => { let mapped = 0; for (const { label, source } of corpus) { @@ -167,9 +327,60 @@ describeNative.each([ expect(mapped).toBeGreaterThan(100_000); }); - it('the copied markdown is exactly the source slice the selection mapped to', () => { + /** + * WHAT THIS PROVES, AND WHAT IT DOES NOT. + * + * It proves that copy SURVIVES the corpus: `buildCopyPayload` re-parses and + * re-projects every slice, and a slice that cuts a construct in half — the + * majority of the selections swept here — is exactly the input that could + * throw or hang. It also pins `markdown` as a pure slice, which is the one + * thing about it a future change could quietly alter. + * + * IT ALSO PROVES THAT NOTHING SWEPT IS LOST. For every one of those + * selections it takes the characters the run actually showed for real source + * (`sourceBackedSlice`), reduces both sides to letters and digits, and + * requires the swept ones to appear in `payload.plain` IN ORDER. Copy is + * allowed to give back more than was swept — a selection that covers a + * construct whole gets its markers and syntax back, and reparsing renders + * them — and it is allowed to spell punctuation and whitespace differently. + * It is not allowed to drop a letter. That is what a paste has to be worth, + * and it is the assertion an offset regression fails: shifting every mapped + * span by ONE character (the shape of the bug the linear-piece test above + * exists for) turns 0 violations into 63,388. + * + * Three enumerated exclusions, each of them "the slice means something else + * standing alone" rather than a hole in the property: synthetic glyphs are + * removed from the swept side rather than looked for in the copy + * (`sourceBackedSlice`); a selection touching verbatim text is skipped + * (`verbatimRanges`); and so is a slice that opens a block it does not + * finish (`OPENS_AN_UNFINISHABLE_BLOCK`). What is left is ~89,000 selections + * under llmChat and ~85,000 under everything, all of them clean. + * + * It does NOT prove the reparse property that copy once claimed ("markdown + * re-parses to the same visible text as the selection") — subsequence is + * weaker than equality, and today only 57,333 of the 136,365 selections + * swept under `llmChat` come back character-identical (54,343 of 129,861 + * under `everything`). Under the bounds the test above establishes, the + * `markdown` equality here restates `doc.source.slice(...)` back at itself + * and cannot fail. And the equality property itself is false for PARTIAL + * selections: a block's own syntax — a heading's + * `# `, a quote's `> `, a list marker, a fence — projects no text, so it + * belongs to no piece, and only a selection that covers the whole construct + * gets it back (`mapSelectionToSource` unions the covering + * `ProjectedExtent`s into the hull). So selecting the whole of + * `1. first\n2. second` does copy `1. first\n2. second` and re-parses to + * the same list — while a selection that starts one character into the + * first item copies `first\n2. second`, which re-parses to one paragraph. + * That loss is measured rather than asserted away — see the round-trip + * census below, which counts whole-block copies precisely because those are + * the ones the extents can save. + */ + it('copy returns exactly the mapped source slice, and loses nothing swept', () => { + let copies = 0; + let checked = 0; for (const { label, source } of corpus) { const doc = parseDocument(source, options); + const verbatim = verbatimRanges(doc); for (const { projected } of proseRuns(doc)) { const offsets = selectionOffsets(projected.text.length); for (const start of offsets) { @@ -178,16 +389,235 @@ describeNative.each([ const span = mapSelectionToSource(projected, { start, end }); if (span === null) continue; const payload = buildCopyPayload(doc, span, { options }); + copies++; if (payload.markdown !== doc.source.slice(span.start, span.end)) { throw new Error( `${label}: copy payload markdown is not the source slice for ` + `${JSON.stringify(span)}`, ); } + if (typeof payload.plain !== 'string') { + throw new Error(`${label}: copy payload has no plain text`); + } + if ( + verbatim.some((r) => r.start < span.end && r.end > span.start) || + OPENS_AN_UNFINISHABLE_BLOCK.test(payload.markdown) + ) { + continue; + } + checked++; + const swept = lettersAndDigits( + sourceBackedSlice(projected, start, end), + ); + if (!isSubsequence(swept, lettersAndDigits(payload.plain))) { + throw new Error( + `${label}: selecting [${start},${end}) showed ` + + `${JSON.stringify(projected.text.slice(start, end))} but the ` + + `copy of ${JSON.stringify(span)} reads ` + + `${JSON.stringify(payload.plain)}, which does not carry every ` + + 'letter that was swept', + ); + } } } } } + expect(copies).toBeGreaterThan(100_000); + // The exclusions must not be what carries the test: the great majority of + // the sweep is still subject to the subsequence property. + expect(checked).toBeGreaterThan(80_000); + }); + + /** + * A LINEAR PIECE SHOWS ITS OWN SOURCE — the invariant every offset in the + * library rests on, and the one the fixtures cannot check. + * + * `mapSelectionToSource` maps a piece whose display length equals its source + * length code-unit for code-unit. That arithmetic is only meaningful if the + * piece is pinned to the source the display actually came from, and a piece + * can be linear and WRONG: a fenced block whose body also occurs inside its + * info string (```` ```js\njs\n``` ````) used to pin to the fence line — + * same length, so every offset in the block mapped one construct to the + * left and nothing downstream noticed. The check is character-level and the + * exceptions are enumerated: the projection is allowed the one-for-one + * substitutions in `PROJECTED_SUBSTITUTIONS` — soft breaks, both smart + * quotes, NUL, and an indented block's leftover indent — and no others. + */ + it('every linear piece displays the source it is pinned to', () => { + let linear = 0; + let nonLinear = 0; + let indivisibleSource = 0; + for (const { label, source } of corpus) { + const doc = parseDocument(source, options); + for (const { projected } of proseRuns(doc)) { + for (const piece of projected.pieces) { + if (piece.source === null) continue; + const text = projected.text.slice(piece.textStart, piece.textEnd); + const src = doc.source.slice(piece.source.start, piece.source.end); + // A non-linear piece is indivisible by design (decoded entity, alt + // text, an embed placeholder) — nothing claims its text is its + // source. + if (text.length !== src.length) { + nonLinear++; + indivisibleSource += src.length; + continue; + } + linear++; + for (let i = 0; i < src.length; i++) { + if (src[i] === text[i]) continue; + const allowed = PROJECTED_SUBSTITUTIONS.get(src[i]); + if (allowed !== undefined && allowed.includes(text[i])) continue; + throw new Error( + `${label}: linear piece ${JSON.stringify(piece)} shows ` + + `${JSON.stringify(text)} for source ${JSON.stringify(src)}`, + ); + } + } + } + } + expect(linear).toBeGreaterThan(1_000); + // A CEILING ON WHAT IS PINNED INDIVISIBLY, because "correct" is not the + // only thing that matters here: a piece whose display length differs from + // its source length cannot be subdivided, so every selection touching it + // copies the whole thing. The measure is SOURCE CHARACTERS, not pieces, + // and there is deliberately NO ceiling on the piece count — splitting one + // 96-character indivisible piece into three small ones raises the count + // and is exactly the improvement wanted, so a count gates the wrong + // direction. (One stood here for a while at `nonLinear < 110`, eleven + // above the 99 the corpus produces, contradicting this paragraph directly + // above it: the next split would have turned the suite red for getting + // better. `nonLinear` is still counted, because it is what makes the + // character total legible in a failure.) + // + // `alignLiteral` covers a diverged literal with linear runs wherever the + // source still spells the display (an escaped `\*`, an `&`) and pins + // only the respelled stretch itself (`…`, `--`, an image's alt + // text) — where a one-for-one respelling like a smart quote even stays + // linear. Today that is 603 source characters over 90 pieces under + // llmChat and 628 over 99 under everything; before the literal was + // covered piecewise, a single escape made the enclosing PARAGRAPH one + // indivisible piece, and before `nextResync` learned to hold the source + // cursor still (mapSelection.ts) it was roughly twice this — a literal + // whose display carries characters the slice never had, an indented code + // block's synthesized indent above all, fell back to a whole-span pin. + if (indivisibleSource >= 700) { + throw new Error( + `${indivisibleSource} source characters are pinned indivisibly, over ` + + `${nonLinear} non-linear pieces — the ceiling is 700 characters, and ` + + 'there is none on the piece count', + ); + } + }); + + /** + * A WHOLE-CONSTRUCT SELECTION COPIES THE WHOLE CONSTRUCT — the property + * `ProjectedExtent` exists for, held over every syntax-bearing block in the + * corpus. + * + * A block's own syntax projects no text: a heading's `# `, a quote's `> `, a + * list item's marker, a fence, a table's pipes. It therefore belongs to no + * piece, and the hull of the pieces a selection touched used to be the whole + * answer — so selecting a whole list and copying it yielded `one\n- two`, + * which re-parses as a paragraph followed by a one-item list. The mapped + * span must now CONTAIN the block's own source span whenever the selection + * covers the block's whole projected range. + * + * Containment rather than equality, because the property under test is that + * nothing is LOST: a hull wider than the block copies more context than the + * user swept, which is at worst untidy, while a narrower one drops the + * markers and changes what the paste means. Every block in the corpus in + * fact lands on equality today (checked by tightening this to `!==` and + * running it), so containment is headroom rather than slack — the case that + * used to need it, an indented code block whose literal fell back to a + * whole-span pin, is gone with `nextResync`'s held candidate in + * src/selection/mapSelection.ts. + */ + it('a whole-block selection maps to a span covering that block', () => { + const syntaxBearing = new Set([ + 'heading', + 'blockquote', + 'list', + 'codeBlock', + 'table', + ]); + let checked = 0; + for (const { label, source } of corpus) { + const doc = parseDocument(source, options); + for (const { run } of proseRuns(doc)) { + for (const block of run.blocks) { + if (!syntaxBearing.has(block.kind)) continue; + const projected = projectRun({ ...run, blocks: [block] }, doc); + if (projected.text.length === 0) continue; + const span = mapSelectionToSource(projected, { + start: 0, + end: projected.text.length, + }); + if (span === null) continue; + checked++; + if (span.start > block.span.start || span.end < block.span.end) { + throw new Error( + `${label}: selecting the whole of a ${block.kind} mapped to ` + + `${JSON.stringify(span)}, which does not cover the block's own ` + + `span ${JSON.stringify(block.span)} — its markers would be ` + + 'lost from the copy', + ); + } + } + } + } + // The corpus really is full of headings, lists, quotes, fences and + // tables, so a regression cannot pass by having nothing left to check. + expect(checked).toBeGreaterThan(200); + }); + + /** + * THE ROUND-TRIP CENSUS: the reparse property, measured. + * + * "Copy this block and paste it somewhere else" is the gesture copy exists + * for, so this walks every block of every prose run in the corpus, selects + * all of its projected text, and asks whether the copied markdown shows the + * same characters again. It cannot be an assertion — the property is still + * false for whole classes of block, and the counts below say by how much: + * + * - a slice can mean something else on its own: CommonMark example 65's + * paragraph text is literally `## foo`, which re-parses as a heading; + * - a REFERENCE link or image points at a definition somewhere else in the + * document, which no slice of one block can carry; + * - an indented fence loses its relative indent, because the slice starts + * at the fence rather than at the line; + * - a trailing newline or trailing spaces do not survive a re-parse. + * + * What is no longer on that list is the big one: a block's own syntax used + * to fall outside the hull, so a list copied without its markers, a quote + * without its `> ` and a fenced block without its fences. `ProjectedExtent` + * put those back (see the test above), and the count went from 493 of 761 to + * 696 — the floor below is what holds it there. A change that makes copy + * lossier — a piece pinned to the wrong source, a hull that stops covering + * what the user swept — drops the count and fails here. + */ + it('records how many whole-block copies come back as themselves', () => { + let blocks = 0; + let roundTrips = 0; + for (const { source } of corpus) { + const doc = parseDocument(source, options); + for (const { run } of proseRuns(doc)) { + for (const block of run.blocks) { + const projected = projectRun({ ...run, blocks: [block] }, doc); + if (projected.text.length === 0) continue; + const span = mapSelectionToSource(projected, { + start: 0, + end: projected.text.length, + }); + if (span === null) continue; + blocks++; + if (buildCopyPayload(doc, span, { options }).plain === projected.text) { + roundTrips++; + } + } + } + } + expect(blocks).toBeGreaterThan(700); + expect(roundTrips).toBeGreaterThan(680); }); }); @@ -394,13 +824,28 @@ describeNative.each([ JSON.stringify(span), ); } - const payload = buildCopyPayload(doc, span, { options }); + const payload = buildCopyPayload(doc, span, { + options, + // The lookup belongs in the copy context for the same reason it + // belongs in the projection: without it, `plain` would project + // every claimed node's own text where the screen shows one + // placeholder. It is offered nodes from the reparse of this + // slice, so it has to claim by shape — which is what makes + // `node.kind === 'link'` the right kind of claim to sweep with. + embed: claimLinks, + }); if (payload.markdown !== doc.source.slice(span.start, span.end)) { throw new Error( `${label}: embed copy payload is not the source slice for ` + JSON.stringify(span), ); } + if (payload.plain.includes('\ufffc')) { + throw new Error( + `${label}: embed copy payload left a raw placeholder in plain: ` + + JSON.stringify(payload.plain), + ); + } } } } diff --git a/conformance/streaming/prefix-oracle.test.ts b/conformance/streaming/prefix-oracle.test.ts index ff3c30b..9173d7e 100644 --- a/conformance/streaming/prefix-oracle.test.ts +++ b/conformance/streaming/prefix-oracle.test.ts @@ -40,10 +40,12 @@ import { visit } from '../../src/document/visit'; import type { Engine } from '../../src/engine/Engine'; import { parseDocument } from '../../src/engine/Engine'; import type { EngineOptions } from '../../src/engine/options'; -import { presets } from '../../src/engine/options'; +import { DEFAULT_LINK_PREFIXES, presets } from '../../src/engine/options'; import { describeNative, requireNativeEngine } from '../../src/engine/native/__tests__/support'; import { trimTrailingPlaceholders } from '../../src/stream/placeholders'; +import type { BufferScheduler, IdleScheduler } from '../../src/stream/StreamSession'; import { StreamSession } from '../../src/stream/StreamSession'; +import { createSmoother } from '../../src/stream/smoothing'; const FIXTURE_DIR = path.resolve(__dirname, '..', 'fixtures'); const PER_FIXTURE_TIMEOUT_MS = 120_000; @@ -123,6 +125,43 @@ function expectedDisplay(doc: ParsedDocument, settledUntil: number): Block[] { ]; } +/** + * How the deltas reach the session. + * + * `'append'` is the direct entry point. `'buffered'` is the one an app + * actually uses for a token stream — `appendBuffered` with a frame + * scheduler, an idle scheduler, a `holdBackChars` tail and a `smoother` + * metering the release — and until this existed, the whole coalescing path + * had only ever run against a toy paragraph engine over bare prose (see + * `src/stream/buffering.test.ts`). It matters here because holdback and + * smoothing cut the stream at offsets nothing else picks: a flush commits + * "everything up to 19 characters into the middle of a table row", which is a + * prefix the per-code-point sweep never produces, and every one of those + * commits faces the same fresh-parse oracle. + */ +type FeedMode = 'append' | 'buffered'; + +/** Manual stand-in for a frame/idle scheduler: fires only when told to. */ +function manualScheduler() { + let next: (() => void) | null = null; + return { + schedule(flush: () => void): () => void { + next = flush; + return () => { + next = null; + }; + }, + /** Fires the pending callback (clearing it first, so a re-schedule from + * inside the flush survives). Returns false when nothing was armed. */ + fire(): boolean { + const f = next; + next = null; + f?.(); + return f !== null; + }, + }; +} + interface OracleResult { readonly final: ReturnType; readonly snapshotCount: number; @@ -150,6 +189,7 @@ function streamDeltas( options: EngineOptions | undefined, engine: Engine, verifyPrefixes = true, + mode: FeedMode = 'append', ): OracleResult { let snapshotCount = 0; let spoilerSnapshots = 0; @@ -158,7 +198,26 @@ function streamDeltas( // Once a block has entered the settled prefix, every subsequent snapshot // must contain the very same object (===) for that kind+span. const settledSeen = new Map(); - const session = new StreamSession({ options, engine }); + const frame = manualScheduler(); + const idle = manualScheduler(); + // One frame per delta at 1200cps releases ~19 units a flush: fast enough + // that the buffer tracks the stream rather than pooling the whole fixture, + // slow enough that most flushes commit a partial construct. + let clock = 0; + const bufferScheduler: BufferScheduler = (flush) => frame.schedule(flush); + const idleScheduler: IdleScheduler = (flush) => idle.schedule(flush); + const session = + mode === 'append' + ? new StreamSession({ options, engine }) + : new StreamSession({ + options, + engine, + bufferScheduler, + idleScheduler, + holdBackChars: 4, + now: () => clock, + smoother: createSmoother({ charsPerSecond: 1200, now: () => clock }), + }); const unsubscribe = session.subscribe((snap) => { snapshotCount += 1; @@ -198,7 +257,27 @@ function streamDeltas( } }); - for (const delta of deltas) session.append(delta); + if (mode === 'append') { + for (const delta of deltas) session.append(delta); + } else { + for (const delta of deltas) { + session.appendBuffered(delta); + clock += 16; + frame.fire(); + } + // Play the metered tail out at the same cadence instead of letting + // finalize dump it: the drain's own flushes are prefixes too, and the + // last few characters come back through the idle drain past the + // holdback. + for (let guard = 0; session.pendingLength > 0 && guard < 20_000; guard += 1) { + clock += 16; + if (!frame.fire() && !idle.fire()) break; + } + } + // Deliberately `length`, not `length + pendingLength`: in buffered mode the + // loop above must have played the whole buffer out through real flushes, so + // a holdback or a smoother that stranded text fails here rather than being + // covered up by finalize's drain. const fedLength = session.length; session.finalize('end'); unsubscribe(); @@ -222,6 +301,32 @@ function codePoints(text: string): string[] { describeNative('streaming prefix oracle', () => { const engine = (): Engine => requireNativeEngine(); + /** + * Option sets the whole fixture corpus is swept under. + * + * `presets.llmChat` is what the package ships. The other two are the + * options that DECOUPLE a text node's value from its source slice, which is + * precisely the condition the parse-free fast path stands down on + * (`if (text.value !== raw) return false`, `StreamSession.tryFastPath`): + * `smartPunctuation` turns `"` into curly quotes and `--` into an en dash, + * so value and source differ in length as well as content, and + * `html: 'raw'` is the only mode that emits htmlBlock/htmlInline nodes at + * all — nodes carrying source literals the splice has to rebase. Neither + * had a single prefix case before, which left the guard that exists for + * them ungated. + */ + const FIXTURE_OPTION_SETS: ReadonlyArray<{ + name: string; + options: EngineOptions; + }> = [ + { name: 'llmChat', options: presets.llmChat }, + { + name: 'smartPunctuation', + options: { ...presets.llmChat, smartPunctuation: true }, + }, + { name: "html:'raw'", options: { ...presets.llmChat, html: 'raw' } }, + ]; + test('fixture directory has the expected corpus', () => { expect(fixtureFiles.length).toBeGreaterThanOrEqual(6); expect(TRANSCRIPT.deltas.length).toBeGreaterThan(50); @@ -235,31 +340,67 @@ describeNative('streaming prefix oracle', () => { describe(file, () => { const full = fs.readFileSync(path.join(FIXTURE_DIR, file), 'utf8'); + for (const { name, options } of FIXTURE_OPTION_SETS) { + test( + `every prefix splices to what a fresh native parse of the same source displays (${name})`, + () => { + const result = streamDeltas(codePoints(full), options, engine()); + + expect(result.fedLength).toBe(full.length); + expect(result.final.phase).toBe('settled'); + expect(result.final.document.source).toBe(full); + expect(result.prefixViolations).toEqual([]); + // Frozen blocks must keep referential identity in every later + // snapshot, including the one finalize produces — and the check + // must not be vacuous: every fixture is multi-block, so blocks do + // settle mid-stream. + expect(result.settledBlocksSeen).toBeGreaterThan(0); + expect(result.identityViolations).toEqual([]); + // Spoilers are off in all three option sets, and the transform + // runs after the engine on every reparse — so a spoiler appearing + // at ANY prefix would mean the option leaked, not that the final + // document is wrong. Checked per snapshot for that reason. + expect(result.spoilerSnapshots).toBe(0); + expect(countSpoilers(result.final.document)).toBe(0); + + // Deep equality includes the ABSENCE of incomplete/synthetic: a + // fresh parse never carries them, so a surviving repair marker + // fails here. + expect(result.final.document).toEqual( + parseDocument(full, options, engine()), + ); + }, + PER_FIXTURE_TIMEOUT_MS, + ); + } + test( - 'every prefix splices to what a fresh native parse of the same source displays', + 'the buffered entry point splices the same way, holdback and smoothing included', () => { - const result = streamDeltas(codePoints(full), presets.llmChat, engine()); + // Same corpus, same oracle, fed the way an app feeds a token + // stream: appendBuffered, a 4-character holdback, a metered + // release and an idle drain for the tail. The commits land on + // different prefixes than the per-code-point sweep produces, and + // the run has to end with every character appended — a holdback + // that stranded its tail, or a smoother that lost a cut, shows up + // as a short `fedLength` here rather than in production. + const result = streamDeltas( + codePoints(full), + presets.llmChat, + engine(), + true, + 'buffered', + ); expect(result.fedLength).toBe(full.length); - expect(result.final.phase).toBe('settled'); expect(result.final.document.source).toBe(full); expect(result.prefixViolations).toEqual([]); - // Frozen blocks must keep referential identity in every later - // snapshot, including the one finalize produces — and the check - // must not be vacuous: every fixture is multi-block, so blocks do - // settle mid-stream. - expect(result.settledBlocksSeen).toBeGreaterThan(0); expect(result.identityViolations).toEqual([]); - // Spoilers are off in every preset, and the transform runs after the - // engine on every reparse — so a spoiler appearing at ANY prefix - // would mean the option leaked, not that the final document is - // wrong. Checked per snapshot for that reason. - expect(result.spoilerSnapshots).toBe(0); - expect(countSpoilers(result.final.document)).toBe(0); - - // Deep equality includes the ABSENCE of incomplete/synthetic: a - // fresh parse never carries them, so a surviving repair marker - // fails here. + expect(result.settledBlocksSeen).toBeGreaterThan(0); + // Coalescing means fewer commits than characters — otherwise the + // buffered path would be exercising nothing the append path does + // not. + expect(result.snapshotCount).toBeLessThan(full.length); expect(result.final.document).toEqual( parseDocument(full, presets.llmChat, engine()), ); @@ -332,6 +473,130 @@ describeNative('streaming prefix oracle', () => { PER_FIXTURE_TIMEOUT_MS, ); + test( + 'a streamed spoiler never paints its body in the clear', + () => { + // The tail repair closes an unpaired '||' so `applySpoilers` has a + // pair to work with. Without it the transform sees one marker, returns + // the paragraph untouched, and the hidden text renders as ordinary + // prose in every snapshot from the second pipe until the closing run + // arrives — the one construct whose whole job is to not be read. + const src = 'The answer is ||hunter2|| and nothing else.\n'; + const secretStart = src.indexOf('hunter2'); + const secretEnd = secretStart + 'hunter2'.length; + const options: EngineOptions = { + extensions: { ...ALL_ON.extensions, spoilers: true }, + }; + + const leaks: string[] = []; + let covered = 0; + const session = new StreamSession({ options, engine: engine() }); + const unsubscribe = session.subscribe((snap) => { + const hidden: { start: number; end: number }[] = []; + visit(snap.document, (n) => { + if (n.kind === 'spoiler') hidden.push(n.span); + }); + let sawSecret = false; + visit(snap.document, (n) => { + if (n.kind !== 'text') return; + if (n.span.start >= secretEnd || n.span.end <= secretStart) return; + sawSecret = true; + const inside = hidden.some( + (h) => h.start <= n.span.start && h.end >= n.span.end, + ); + if (!inside) { + leaks.push(`rev ${snap.revision}: ${JSON.stringify(n.value)} outside every spoiler`); + } + }); + if (sawSecret) covered += 1; + }); + for (const cp of codePoints(src)) session.append(cp); + session.finalize('end'); + unsubscribe(); + + expect(leaks).toEqual([]); + // Non-vacuous: the secret really was in the document for most of the + // stream, and it ends up hidden. + expect(covered).toBeGreaterThan(5); + expect(countSpoilers(session.snapshot().document)).toBe(1); + + // And the splice still agrees with a fresh parse at every prefix. + const result = streamDeltas(codePoints(src), options, engine()); + expect(result.prefixViolations).toEqual([]); + expect(result.identityViolations).toEqual([]); + expect(result.spoilerSnapshots).toBeGreaterThan(0); + expect(result.final.document).toEqual(parseDocument(src, options, engine())); + }, + PER_FIXTURE_TIMEOUT_MS, + ); + + test.each([ + [ + 'a paragraph', + 'The answer is:\n\n||42 is the answer|| and nothing else.\n', + '42 is the answer', + ], + [ + 'a list item', + 'Answers:\n\n- ||hunter2|| first\n- plain second\n', + 'hunter2', + ], + ])( + 'a spoiler opening its line never paints its body in the clear (%s)', + (_shape, src, secret) => { + // The repair used to stand down on any line whose first character is a + // pipe, on the theory that it might be a table row — which is exactly + // what a spoiler opening its own line looks like, so the body leaked + // for the whole stream in the commonest shape there is. Table + // membership is now decided the way the parse decides it: a delimiter + // row under a header line. + const secretStart = src.indexOf(secret); + const secretEnd = secretStart + secret.length; + const options: EngineOptions = { + extensions: { ...ALL_ON.extensions, spoilers: true }, + }; + + const leaks: string[] = []; + let covered = 0; + const session = new StreamSession({ options, engine: engine() }); + const unsubscribe = session.subscribe((snap) => { + const hidden: { start: number; end: number }[] = []; + visit(snap.document, (n) => { + if (n.kind === 'spoiler') hidden.push(n.span); + }); + let sawSecret = false; + visit(snap.document, (n) => { + if (n.kind !== 'text') return; + if (n.span.start >= secretEnd || n.span.end <= secretStart) return; + sawSecret = true; + const inside = hidden.some( + (h) => h.start <= n.span.start && h.end >= n.span.end, + ); + if (!inside) { + leaks.push( + `rev ${snap.revision}: ${JSON.stringify(n.value)} outside every spoiler`, + ); + } + }); + if (sawSecret) covered += 1; + }); + for (const cp of codePoints(src)) session.append(cp); + session.finalize('end'); + unsubscribe(); + + expect(leaks).toEqual([]); + expect(covered).toBeGreaterThan(5); + expect(countSpoilers(session.snapshot().document)).toBe(1); + + // …and the splice still agrees with a fresh parse at every prefix. + const result = streamDeltas(codePoints(src), options, engine()); + expect(result.prefixViolations).toEqual([]); + expect(result.identityViolations).toEqual([]); + expect(result.final.document).toEqual(parseDocument(src, options, engine())); + }, + PER_FIXTURE_TIMEOUT_MS, + ); + const ALL_ON: EngineOptions = { extensions: { tables: true, @@ -368,6 +633,186 @@ describeNative('streaming prefix oracle', () => { PER_FIXTURE_TIMEOUT_MS, ); + test( + 'raw HTML blocks splice the same way, blank lines and all', + () => { + // `html: 'raw'` is the only mode that produces htmlBlock nodes at all, + // and nothing else in this sweep exercises it. CommonMark HTML blocks + // of types 1-5 (`\n\n', + '
\n type 6 ends at the blank line\n
\n\n', + '\n\n', + 'Closing prose.\n', + ].join(''); + const result = streamDeltas(codePoints(full), options, engine()); + expect(result.prefixViolations).toEqual([]); + expect(result.identityViolations).toEqual([]); + expect(result.settledBlocksSeen).toBeGreaterThan(0); + expect(result.final.document).toEqual( + parseDocument(full, options, engine()), + ); + }, + PER_FIXTURE_TIMEOUT_MS, + ); + + test( + 'bare ftp autolinks and list-item fences splice the same way', + () => { + // Two shapes no fixture contains. md4c permissive-autolinks ftp as + // well as http/https, so an ftp tail must stand the fast path down + // (allowlisted here, or the autolink would degrade to text and hide + // the divergence); and a fence opened on a list-marker line must be + // read as an opener, or its indented closer looks like one and the + // anchor never advances again. + const options: EngineOptions = { + ...presets.llmChat, + urlPolicy: { linkPrefixes: [...DEFAULT_LINK_PREFIXES, 'ftp://'] }, + }; + const full = [ + 'Mirrors live at ftp://example.com/pub and https://example.com/pub.\n\n', + '- ```sh\n curl ftp://example.com/pub/file.txt\n ```\n', + '- and a second item with a _partial word\n\n', + 'Closing prose after the list.\n', + ].join(''); + const result = streamDeltas(codePoints(full), options, engine()); + expect(result.prefixViolations).toEqual([]); + expect(result.identityViolations).toEqual([]); + expect(result.settledBlocksSeen).toBeGreaterThan(0); + expect(result.final.document).toEqual( + parseDocument(full, options, engine()), + ); + }, + PER_FIXTURE_TIMEOUT_MS, + ); + + test( + 'a spoiler-enabled sweep splices the same way at every prefix', + () => { + // ALL_ON pins spoilers OFF so the corpus can assert their absence. + // Spoilers are the one extension applied as a post-engine transform + // over the decoded tree, so they are also the one whose result the + // splice could disagree with — this runs the same sweep with them on, + // over text that has both a real spoiler and a `||` that must stay + // literal because its partner never arrives. + const options: EngineOptions = { + extensions: { ...ALL_ON.extensions, spoilers: true }, + }; + const full = [ + 'Intro paragraph before anything hidden.\n\n', + 'The answer is ||hunter2|| and the runner-up is ||nobody||.\n\n', + '- a list item with ||a hidden phrase|| inside\n', + '- and one with a lone || that never closes\n\n', + '| col | value |\n| :- | -: |\n| a | ||secret|| |\n\n', + 'Closing prose.\n', + ].join(''); + const result = streamDeltas(codePoints(full), options, engine()); + expect(result.prefixViolations).toEqual([]); + expect(result.identityViolations).toEqual([]); + expect(result.settledBlocksSeen).toBeGreaterThan(0); + // Non-vacuous: spoilers really were in the streamed snapshots. + expect(result.spoilerSnapshots).toBeGreaterThan(0); + const fresh = parseDocument(full, options, engine()); + expect(countSpoilers(result.final.document)).toBe(countSpoilers(fresh)); + expect(countSpoilers(fresh)).toBeGreaterThan(0); + expect(result.final.document).toEqual(fresh); + }, + PER_FIXTURE_TIMEOUT_MS, + ); + + test( + 'a link reference definition arriving after a settled paragraph splices the same way', + () => { + // The one construct that acts at a distance: the definition at the end + // turns the `[foo]` in the FIRST paragraph — frozen by the anchor long + // before it arrives — into a resolved reference link, and the `[bar]` + // typed after it must resolve against a definition that sits below the + // anchor. Nothing in the corpus contains one, and both directions used + // to leave the snapshot (and the finalized document) showing literal + // text where a fresh parse shows a link. + const full = [ + 'See [foo] here.\n\n', + 'A middle paragraph that settles.\n\n', + '[foo]: https://example.com/foo\n', + '[bar]: https://example.com/bar "titled"\n\n', + 'And [bar] afterwards.\n', + ].join(''); + const result = streamDeltas(codePoints(full), presets.llmChat, engine()); + expect(result.prefixViolations).toEqual([]); + expect(result.final.document).toEqual( + parseDocument(full, presets.llmChat, engine()), + ); + // Identity is the one thing a definition genuinely costs, and only + // where it has to: the first paragraph's parse CHANGED, so it cannot + // be the same object any more. Every other settled block keeps its + // identity — the session drops the anchor, not the identity cache, + // and `remember` re-checks structure before reusing an entry. + expect( + result.identityViolations.map((v) => v.replace(/^rev \d+: /, '')), + ).toEqual(['settled block paragraph:0:15 lost referential identity']); + // Non-vacuous: both references really did resolve. + let links = 0; + visit(result.final.document, (n) => { + if (n.kind === 'link') links += 1; + }); + expect(links).toBe(2); + // And the buffered path reaches the same place. + const buffered = streamDeltas( + codePoints(full), + presets.llmChat, + engine(), + true, + 'buffered', + ); + expect(buffered.prefixViolations).toEqual([]); + expect(buffered.final.document).toEqual(result.final.document); + }, + PER_FIXTURE_TIMEOUT_MS, + ); + + test( + 'a definition behind a list item content indent splices the same way', + () => { + // The same construct written where md4c also honours it: at the + // content column of a list item, with no marker of its own. A scan + // that only stripped container MARKERS read those four spaces as + // indented code, kept the first paragraph frozen as literal text, and + // handed back a prefix a fresh parse disagrees with — and it did so + // only for some delta granularities, since a line first seen as ' ' + // took a different path through the scan. + const full = [ + 'See [foo] here.\n\n', + '- outer\n - inner\n\n', + ' [foo]: https://example.com/foo\n\n', + 'And more prose afterwards.\n', + ].join(''); + for (const deltas of [ + codePoints(full), + full.split(/(?<=\n)/), + [full], + ]) { + const result = streamDeltas(deltas, presets.llmChat, engine()); + expect(result.prefixViolations).toEqual([]); + expect(result.final.document).toEqual( + parseDocument(full, presets.llmChat, engine()), + ); + } + // Non-vacuous: the reference really did resolve. + let links = 0; + visit(parseDocument(full, presets.llmChat, engine()), (n) => { + if (n.kind === 'link') links += 1; + }); + expect(links).toBe(1); + }, + PER_FIXTURE_TIMEOUT_MS, + ); + test( 'a streamed table splices the same way', () => { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f4e7b21..f6a1618 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -14,7 +14,7 @@ markdown source (static string, or accumulated stream text) v engine/ Engine interface: the pluggable parser native/ md4c behind one ArrayBuffer per parse (default, and the only parser) - entities.ts, urlPolicy.ts shared decode + URL policy + entities.ts, urlPolicy.ts fallback entity decode + URL policy extensions/spoilers.ts opt-in post-parse transform v document/ ParsedDocument; every node: { kind, span: SourceSpan } @@ -27,13 +27,14 @@ selection/ segmentRuns: adjacent flowing blocks -> selectable runs v view/ SelectableMarkdown runs -> RunHost (native SelectableRunHost; required, throws where not linked) - standalone blocks (classifyBlock) -> block renderers + standalone blocks (image, spoiler, a classifyBlock claim) -> block renderers, + nesting capped at MAX_RENDER_DEPTH (64); deeper subtrees render flat text per-run memoization keyed on span + settled identity v onSelectionCopy({ action, plain, markdown, span }) <- selection offsets (UTF-16) from the host ``` -Pacing (`appendBuffered`, holdback, smoothers) sits in front of the stream layer and feeds the same `append` path, so nothing below it knows coalescing exists. The stream layer sits in front of the engine: tail repair is a text-to-text transform, so any engine gets artifact-free streaming for free. Selection never touches the render tree; it maps display offsets to source spans through the piece map, and the React tree is only a projection. +Pacing (`appendBuffered`, holdback, smoothers) sits in front of the stream layer and feeds the same `append` path, so nothing below it knows coalescing exists. The stream layer sits in front of the engine: tail repair is a text-to-text transform, and the constructs it completes are CommonMark's. Selection never touches the render tree; it maps display offsets to source spans through the piece map, and the React tree is only a projection. ## Three rules the design follows @@ -50,7 +51,7 @@ An engine that cannot produce exact spans cannot back this library. Span correct ### 2. Streaming is incremental, not a re-render loop -`StreamSession` owns the accumulated text. After each parse it finds a safe anchor: the end of the last block no future append can change. Each append reparses only `source.slice(anchor)`, rebases the spans (`shiftSpans`, a deep clone), and splices the frozen prefix back in by reference. Settled blocks keep referential identity, so settled runs never re-render, and parse cost tracks the tail rather than the document. A plain-prose delta skips the engine and extends the trailing paragraph. When nothing can anchor yet (one giant list, an unclosed fence), the update falls back to a correct full reparse. +`StreamSession` owns the accumulated text. After each parse it finds a safe anchor: the end of the last block no future append can change. Each append reparses only `source.slice(anchor)`, rebases the spans (`shiftSpans`, a deep clone), and splices the frozen prefix back in by reference. Settled blocks keep referential identity, so an untouched settled run re-renders zero times and the one run that absorbs a newly settled block re-renders once per settle; parse cost tracks the tail rather than the document. A plain-prose delta skips the engine and extends the trailing paragraph. When nothing can anchor yet (one giant list, an unclosed fence), the update falls back to a correct full reparse. Before each parse, `repairTail` virtually completes unfinished constructs (`**bold`, `[link](https://…`) and suppresses lines that would flip earlier structure (a lone `-` about to make a setext heading). On `finalize` the repairs vanish and the document is a plain parse of exactly what arrived. A prefix oracle asserts at every prefix of every fixture that the spliced snapshot equals a fresh parse. Details in [STREAMING.md](STREAMING.md). @@ -58,15 +59,15 @@ Before each parse, `repairTail` virtually completes unfinished constructs (`**bo Every non-CommonMark extension is opt-in (`ExtensionFlags`, all false by default). A stray `|` in model output must not change how a paragraph renders, so `||…||` is off in every preset except `everything`, and even then only balanced pairs parse. -HTML is stripped by default (`html: 'strip'`). URL schemes are allowlisted (links: `https://`, `http://`, `mailto:`; images: `https://`) at parse time, so no renderer can forget. Blocked links degrade to plain text, blocked images to their alt text. +HTML is stripped by default (`html: 'strip'`), which drops the construct *and* the text inside it: an HTML block contributes nothing to the document. The one exception is an inline `
`, which decodes to a `hardBreak` over the tag rather than vanishing between two words. URL schemes are allowlisted (links: `https://`, `http://`, `mailto:`; images: `https://`). `nativeEngine`'s decoder applies the allowlist as it builds each node, so *that* engine can never return a rejected `href` and no renderer can forget; `parseDocument` runs no policy pass over a substituted engine's output, so the view checks again at the navigation boundary — `openUrl` sanitizes and re-tests every href against the document's resolved `urlPolicy.linkPrefixes` before `Linking.openURL`, on the renderer path and the native-run press path alike. Matching folds case over the scheme and authority only, and a consumer prefix that reaches into a path (`myapp://checkout/`) is treated as a scope: a destination that walks back out of it with `..` (raw or percent-encoded once) is refused. Blocked links degrade to plain text, blocked images to their alt text. -A blocked link leaves no node behind, so an app whose model emits in-app identifiers as links cannot render them. `urlPolicy.blockedLinks: 'node'` keeps the node, flagged `blocked: true`, for your `link` renderer. The href still never reaches `openURL`. Autolinks stay outside this: their text is their destination, so there is no label to keep. +A blocked link leaves no node behind, so an app whose model emits in-app identifiers as links cannot render them. `urlPolicy.blockedLinks: 'node'` keeps the node, flagged `blocked: true`. Inside a flowing run that node projects as a `blockedLink` mark and your `link` renderer does not run — renderers draw standalone blocks only. The channels that reach it there are `attributeForMark`, `onLinkPress` and `embed`; `classifyBlock: 'standalone'` is what puts the block back on the renderer path. The href still never reaches `openURL` on any of them. Autolinks and images stay outside this — `blockedLinks` governs inline links only: an autolink's text is its destination, so there is no label to keep, and a blocked image degrades to its alt text under either setting. ## Module map ``` src/ - index.ts public API barrel + index.ts public API barrel: explicit named exports, internals stay on deep paths document/ span.ts SourceSpan + span algebra (length/contains/intersects/slice) nodes.ts Block/Inline unions, ParsedDocument, type guards @@ -74,8 +75,8 @@ src/ engine/ Engine.ts Engine interface + parseDocument() options.ts ExtensionFlags, EngineOptions, resolveOptions, presets - entities.ts entity/escape decoding for hrefs, titles, info strings (internal) - urlPolicy.ts sanitizeUrl + isUrlAllowed, applied at parse time (internal) + entities.ts fallback entity decoding for the decoder's Entity case (internal) + urlPolicy.ts sanitizeUrl + isUrlAllowed, applied at parse time, re-checked at press native.ts re-export facade for the md4c engine native/ md4c binding: protocol.ts, decode.ts, widen.ts, install.ts, index.ts extensions/spoilers.ts applySpoilers(): opt-in post-parse ||…|| transform @@ -85,9 +86,13 @@ src/ shiftSpans.ts span rebasing for the incremental splice smoothing.ts createSmoother()/createAdaptiveSmoother() placeholders.ts trimTrailingPlaceholders() + clusters.ts grapheme-cluster boundaries, so a release cut never splits a glyph selection/ - runs.ts segmentRuns(), classifyBlock() -> RunSegment[] - mapSelection.ts projectRun(), mapSelectionToSource() + runs.ts segmentRuns(), classifyTopLevelBlock() -> RunSegment[] + (classification memoized on block identity; runs break at + DEFAULT_MAX_RUN_CHARS; `liveTail` keeps the streaming tail + in a run of its own) + mapSelection.ts projectRun() (incremental through `previous`), mapSelectionToSource() copy.ts buildCopyPayload(): span -> { markdown, plain } view/ SelectableMarkdown.tsx the component @@ -97,9 +102,15 @@ src/ runDecorations.ts marks + theme -> block chrome the host paints runPressables.ts link marks -> tappable ranges the host hit-tests runEmbeds.ts embed entries -> RunEmbed[] the host reserves space from + imageEmbeds.ts withImageEmbeds(): the built-in image embed claim behind `images` + projectionCache.ts createRunProjectionCache(): incremental projectRun per run + processedColors.ts memoizedProcessColor(): the bounded, evicting processColor memo + runIdentity.ts runKey()/embedRectKey(): identities that survive a settle + selectionRange.ts mapSourceToRunRange(): SourceSpan -> a run's display range; + selectSpanInRuns(): the document-level setSelection walk selectionActions.ts handleSelectionAction(): menu event -> onSelectionCopy payload theme.ts grouped tokens, mergeTheme, defaultTheme/defaultDarkTheme - renderers.tsx default per-kind renderers + override types + renderers.tsx default per-kind renderers + override types + the MAX_RENDER_DEPTH cap agui/ useAgUiSession.ts adapter for one known message (no ag-ui dependency) bindRunTextEvents.ts run-scoped binding over per-message sessions @@ -107,17 +118,25 @@ src/ platform/ cpp/ OffsetParser (md4c SAX -> offset nodes), Protocol.h + FlatBuffer.cpp, SelectableMarkdownJsi (installs the parse global), vendor/md4c - fabric/ shared Fabric C++: measuring shadow node, state, descriptor, measurer facade + fabric/ shared Fabric C++: measuring shadow node, state, descriptor, measurer facade, + android-include/ (the three codegen headers Android's CMake shadows) ios/ UITextView host, Fabric component view, iOS measurer, JSI installer android/ TextView host + ViewManager, CMake/JNI glue, JSI installer. At the package root because RN's Gradle plugin looks for package.json one directory up. native/node/ Node-API harness over the same C++ for tests and benches (never shipped) conformance/ CommonMark runner, streaming prefix oracle, projection oracle, fixtures bench/ Node benchmarks +scripts/ build-node-addon (builds the Node harness), emit-dist-spec-shim (the codegen + spec's dist stub), check-codegen / check-fabric-cpp / check-swift, verify-pack, + release, changelog-section +docs/ this file and its siblings +SelectableMarkdown.podspec, react-native.config.js what autolinking reads; both ship ``` Dependencies point downward: `document` depends on nothing; `engine` on `document`; `stream` on both; `selection` on `document` and `engine`; `view` on all of the above; `agui` on `stream` (plus the `EngineOptions` type from `engine`). Nothing in `src/` imports from `platform/`. +`index.ts` names every export one at a time instead of re-exporting modules wholesale, so the surface is a decision rather than a consequence of where a helper happens to live. It publishes the document model plus its span algebra (`visit`, `findAt`, `childrenOf`, `isBlock`/`isInline`, `spanLength` and friends); `parseDocument`, the `Engine` type, `resolveOptions`/`withOptions`/`presets`, `DEFAULT_LINK_PREFIXES`/`DEFAULT_IMAGE_PREFIXES`, `sanitizeUrl`/`isUrlAllowed`, `applySpoilers`, and the native quartet `nativeEngine`/`createNativeEngine`/`installNativeEngine`/`isNativeEngineAvailable` (with `isNativeEngineInstalled` and `isNativeEnginePermanentlyRefused` for diagnostics); `StreamSession`, `createSmoother`/`createAdaptiveSmoother`/`snapPastLinkDestination`, `repairTail`/`seedFromSettled`/`continueSeed`, `trimTrailingPlaceholders`; `segmentRuns`/`classifyTopLevelBlock`/`DEFAULT_MAX_RUN_CHARS`, `projectRun`/`mapSelectionToSource`/`EMBED_PLACEHOLDER`, `buildCopyPayload`; the ag-ui adapters; and the view layer — `SelectableMarkdown`, `RunHost`, the theme, the renderers with `openUrl`/`textContentOf`/`MAX_RENDER_DEPTH`, the selection-action helpers, and the per-channel resolvers a consumer driving `RunHost` itself needs (`resolveRunAttributes`, `resolveRunPressables`, `resolveRunDecorations`, `resolveRunEmbeds`, `withImageEmbeds`, `createRunProjectionCache`, `mapSourceToRunRange`). What it deliberately does not publish stays reachable one directory in: the flat-buffer decoder and `__linkNativeEngine` at `dist/engine/native`, the selection-menu wire codec at `dist/view/selectionActions`. + One file is build input rather than runtime code. React Native's codegen and babel plugin read `SelectableRunHostNativeComponent.ts` and only match `codegenNativeComponent<…>` in the original source; a `tsc`-transpiled copy yields no view config, the component never registers, and `RunHost` throws for every run. So `package.json` points Metro at `src/index.ts`, `tsconfig.build.json` excludes the file from emit, `RunHost` reaches it through a call-expression `require` (an `import type` would defeat the exclusion), and `npm run check:codegen` asserts the generated C++ still matches. ## Engine interface @@ -163,13 +182,14 @@ parseDocument(md, undefined, plainTextEngine); // or: ``` -Three rules, each a way a real engine goes wrong: +Four rules, each a way a real engine goes wrong: 1. **`source` is the string you were handed.** Copy, selection and streaming all slice that string by the spans below it. Returning a trimmed or normalized source breaks all three. 2. **Spans are absolute UTF-16 offsets into that string.** `match.index` is what makes the paragraph above selectable. Per-chunk offsets like `{ start: 0, end: match[0].length }` render fine and copy the wrong text from the second paragraph on. Nothing checks this for you. 3. **Node kinds are a fixed vocabulary.** `Block` and `Inline` are discriminated unions from `src/document/nodes.ts`: `paragraph`, `heading`, `codeBlock`, `blockquote`, `list`, `listItem`, `table`, `tableRow`, `tableCell`, `thematicBreak`, `htmlBlock`; `text`, `emphasis`, `strong`, `strikethrough`, `underline`, `codeSpan`, `link`, `image`, `autolink`, `hardBreak`, `softBreak`, `math`, `spoiler`, `htmlSpan`. The type checker rejects anything else, and `RendererMap` is a mapped type over `AnyNode['kind']`, so a missing renderer is a compile error. +4. **The stream layer speaks CommonMark, not your dialect.** `StreamSession` never hands you the tail as it arrived. `repairTail` (`src/stream/repair.ts`) appends virtual CommonMark closers (`**bold` parses as `**bold**`) and suppresses a bare trailing construct line that would flip the structure above it (a lone `-` about to make a setext heading), and there is no switch that turns it off. The parse-free fast path calls you at all only when a delta holds one of the characters in `CONSTRUCT_CHARS` (`src/stream/StreamSession.ts`) or ends in a space or tab, so an engine whose own construct characters are not a subset of that class — a `@mention` engine, say — shows stale structure until an unrelated construct character happens to arrive. An engine that is not parsing CommonMark is better driven by parsing the accumulated text yourself than through `StreamSession`. -The second parameter is `ResolvedEngineOptions`, with every default filled in. Honouring the flags is optional; the span invariant is not. +The second parameter is `ResolvedEngineOptions`, with every default filled in. Honouring the flags is optional; the span invariant is not. `options.urlPolicy` in particular is yours to apply: nothing between `parseDocument` and the view re-filters a node's `href`, and the view's press-time re-check is a backstop for navigation only, not for the hrefs your own UI reads off a node. ### The engine that ships @@ -180,13 +200,15 @@ The engine is two halves that meet at the buffer. Which half owns a decision is | | C++ (`platform/cpp/`, before the crossing) | TypeScript (`src/engine/native/`, after it) | | --- | --- | --- | | Structure | md4c decides blocks, inlines and GFM extensions | Replays the events; never re-decides structure | -| Offsets | md4c's UTF-8 byte offsets to UTF-16, the one conversion point | Widens content ranges into construct spans (`widen.ts`) | -| Text | None crosses. Events carry offsets, not characters | Text nodes are slices of the source string JS already holds | +| Offsets | md4c's UTF-8 byte offsets to UTF-16 in `FlatBuffer.cpp`, the one conversion point (`OffsetParser.cpp` only builds the map) | Widens content ranges into construct spans (`widen.ts`) | +| Text | No source *slice* crosses: prose is offsets. Values that are not slices do, in the string table — hrefs, titles, info strings, decoded entities, the U+FFFD standing in for a NUL, and the text md4c synthesizes rather than points at: the `"\n"` it reports for every break and every code- or HTML-block line, which is the largest class, collapsed to one entry per consecutive run by `SaxState::intern` | Text nodes are sliced from the JS source except for those string-table values; escaped runs re-widened and smart punctuation applied here | | Policy | None | URL allowlist and HTML strip/raw, applied while the node is built | | Extensions | md4c flags for GFM, math and underline; never spoilers | Spoilers, opt-in, after the engine returns | -No source text crosses, so cost tracks structure rather than length. Decoding and policy stay in JavaScript because text values are slices of the JS string, and the allowlist has to run where the node is built so `parseDocument` can never return a rejected `href`. `entities.ts` and `urlPolicy.ts` sit at the engine root, not inside `native/`, because they describe the document model; a substituted engine can deep-import them instead of reimplementing the allowlist. Wire format and decoder details are in [NATIVE.md](NATIVE.md). +No source slice crosses, so cost tracks structure rather than length. What stays in JavaScript is escape re-widening, smart punctuation and policy: text values are slices of the JS string, and the allowlist has to run where the node is built so `nativeEngine` can never return a rejected `href` — the view checks a second time at the press, which is what extends the guarantee to a substituted engine. Entity decoding is not on this side: md4c ships the HTML5 table, so it decodes both prose entities and the ones inside an attribute, and `entities.ts` is left holding the decoder's defensive fallback. It and `urlPolicy.ts` sit at the engine root, not inside `native/`, because they describe the document model; `urlPolicy` is exported from the package entry so a substituted engine reuses the allowlist instead of reimplementing it. Wire format and decoder details are in [NATIVE.md](NATIVE.md). ### Headless use from Node -The package entry re-exports the view layer and so imports `react-native`, which plain Node cannot load. Import the deep paths instead: `dist/engine/Engine.js`, `dist/engine/options.js`, `dist/engine/native/index.js`, `dist/stream/StreamSession.js`, `dist/selection/*.js`, `dist/document/*.js`. Parsing still needs md4c, which under Node means the test addon in `native/node/`; see [NATIVE.md](NATIVE.md). A dedicated Node subpath export is a roadmap item. +The package entry re-exports the view layer and so imports `react-native`, which plain Node cannot load. Import the deep paths instead: `dist/engine/Engine.js`, `dist/engine/options.js`, `dist/engine/native/index.js`, `dist/stream/StreamSession.js`, `dist/selection/*.js`, `dist/document/*.js`. Parsing still needs md4c, which under Node means the test addon in `native/node/`; see [NATIVE.md](NATIVE.md). Those are declared subpath exports, not a reach past the package's front door: `package.json`'s `exports` map publishes `./dist/*` both with and without the `.js`, plus an explicit `./dist` entry for the bare directory. Each of those entries carries both conditions — `require` → `./dist/*.js` with types `./dist/*.d.ts`, `import` → `./dist/esm/*.js` with types `./dist/esm/*.d.ts`. The ES modules are the same sources emitted by `tsconfig.esm.json` (`module: es2020`) and finished by `scripts/finish-esm-build.mjs`, which writes `dist/esm/package.json` (`"type": "module"`, `sideEffects: false`) and adds the `.js` extension Node's ESM resolver needs on every relative specifier. They carry no stability promise. + +One caveat on the deep paths: two modules reach the platform through a call-expression `require` — `react-native` in `src/engine/native/install.ts`, the codegen spec in `src/view/RunHost.tsx` — and `require` does not exist in an ES module scope. That is why `require` is listed ahead of `import` there, so a resolver asserting both (Metro) stays on the CommonJS tree; both calls sit in a `try`/`catch` and degrade the way a web bundle already does. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index dac12ff..99396a9 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -2,9 +2,19 @@ Two families of measurement: speed (is it fast, does it stay fast) and accuracy (is the output right, does streaming leave it unchanged). Accuracy -results gate `npm test`, which CI runs on every push and pull request. Speed -results are tracked as trends; the pathological suite becomes a gate when run -with a budget. +results gate `npm test`, which CI runs on every push and pull request. Most +speed results are tracked as trends; three benches are gates. The pathological +suite runs as `npm run bench:pathological -- --require-engine --budget-parse +750 --budget-repair 750 --budget-segment 200 --budget-project 750` in ci.yml's +test job and again in release.yml, and `npm run bench:projection -- +--require-engine` and a budgeted `bench:streaming` over the never-anchoring +transcript run beside it. The budgets are cliff detectors rather than +performance targets — anything that goes quadratic on those inputs lands in +seconds, not in the noise band — and the pathological ones are per stage +because the stages differ by orders of magnitude, so one number loose enough +for the table's parse would let `segment` get a hundred times slower and still +pass. The numbers themselves come from the runs recorded below, not from a CI +runner. There is one parser, md4c through the native module, and every number here is that parser. A machine that cannot build the Node addon measures nothing @@ -20,11 +30,22 @@ The Node benchmarks import compiled output, so build first: npm run build # emits dist/ (tsconfig.build.json) npm run bench:throughput # bench/throughput.mjs npm run bench:pathological # bench/pathological.mjs: report-only by default; - # pass a budget to make it a pass/fail gate: - # npm run bench:pathological -- --budget 2000 -npm run bench:streaming # bench/streaming-replay.mjs + # pass a budget to make it a pass/fail gate, which + # is how CI runs it: + # npm run bench:pathological -- --require-engine \ + # --budget-parse 750 --budget-repair 750 \ + # --budget-segment 200 --budget-project 750 +npm run bench:streaming # bench/streaming-replay.mjs: report-only by + # default; pass a budget to make it a gate, which + # is how CI runs it: + # npm run bench:streaming -- --transcript \ + # conformance/fixtures/transcript-giant-list.json \ + # --require-engine --repeat 1 \ + # --budget-chunk 20 --budget-finalize 50 npm run bench:crossing # bench/crossing.mjs: native parse vs JS decode -npm run bench:all # the four above, in that order +npm run bench:projection # bench/projection.mjs: projected characters per + # document character, cached vs full +npm run bench:all # the five above, in that order # Against other parsers. They are deliberately NOT dependencies of this # package, so point --libs at a directory where you installed them: @@ -34,15 +55,34 @@ npm run bench:headtohead -- --libs /tmp/mdbench # bench/head-to-head.mjs npm run conformance # → conformance/report-native.json npm test # jest: unit suites, the streaming prefix oracle, - # and the selection projection oracle + # the selection projection oracle, and the + # incremental-projection oracle ``` Flags: every timing bench takes `--quick` (one warmup, few iterations; good for checking a bench still runs, useless as a published number). `bench:throughput` and `bench:crossing` take `--iterations N` and -`--replicas R`. `bench:pathological` takes `--budget MS` and `--runs N`. +`--replicas R`. `bench:pathological` takes `--budget MS`, the per-stage +overrides `--budget-parse|-repair|-segment|-project MS` (the global is the +default for any stage without one), `--runs N` and `--require-engine`, which +turns an unresolvable addon from an exit-0 report into a failure. `bench:streaming` takes `--transcript PATH`, `--repeat N`, `--max-chunks N` -and `--replicas N`. `bench:headtohead` takes `--libs DIR`, `--replicas R`, +and `--replicas N`; it replays both pinned transcripts by default, and the +transcript flag narrows it to one. It gates as well as reports. `--budget MS` +is the default for both gated numbers: `--budget-chunk MS` bounds the p99 +append latency, `--budget-finalize MS` the single clean parse `finalize` does, +both per transcript. `--require-engine` turns an unresolvable addon, or a +`StreamSession` that cannot take one character, from an exit-0 report into a +failure. Without a budget nothing fails, because the absolute milliseconds +belong to the machine. A budgeted run that timed nothing (`--max-chunks 0`, +`--repeat 0`) exits 1 — a gate over nothing is a failure, the same rule +`bench:pathological` states. On the machine in Results below, the giant-list +transcript measures chunk p99 0.75–0.86 ms and finalize 0.59–0.85 ms (two runs, +2026-09-03), so the CI budgets sit ~25× and ~60× above them: cliff detectors, +not targets. `bench:projection` takes +`--transcript PATH`, `--chunk N` (delta size, default 18), `--require-engine` +and `--quick` (one document size instead of two — which measures no growth, so +it gates nothing). `bench:headtohead` takes `--libs DIR`, `--replicas R`, `--iterations N` and `--only `. The benches and the conformance runner refuse a stale `--engine` flag with a non-zero exit rather than printing md4c's numbers under a heading you did not choose. @@ -53,14 +93,22 @@ with no C++ toolchain the benches print a notice with the build command and exit 0, and `npm test` skips every suite that parses markdown. CI builds the addon as a hard gate so a job cannot go green having parsed nothing. -## Results (2026-09-01) +## Results (2026-09-02) Apple M2 Max (12-core, 64 GB), macOS 26.5, arm64 Node v22.12.0 running -natively (not under Rosetta), this repo at v0.10.0, addon built by -`scripts/build-node-addon.mjs` from the vendored md4c. One run of each command. -In the August 2026 runs on this machine, medians moved by up to ~20% between -back-to-back runs of the same command, so treat differences inside that band -as noise. +natively (not under Rosetta), this repo at v0.11.0 plus the unreleased audit +pass, addon rebuilt by `scripts/build-node-addon.mjs` from the vendored md4c. +One run of each command. In the August 2026 runs on this machine, medians moved +by up to ~20% between back-to-back runs of the same command, so treat +differences inside that band as noise. + +Everything below is from a 2026-09-02 re-run except two things. The +cross-parser comparison in the next section is still the 2026-09-01 run: the +other parsers are deliberately not dependencies of this package, so re-running +it needs `--libs `. And the streaming-replay rows and the +incremental-vs-full ratio were re-measured on 2026-09-03, on the same machine +and the same arm64 Node, because the bench changed which statistic it reports; +those rows say so. ### Markdown to HTML throughput @@ -93,8 +141,9 @@ re-encoding as failures. The August 2026 run measured this package at 11.37 MB/s on this corpus. This machine's default `node` is an x86_64 build running under Rosetta, and an -x86_64 run today lands in that lower range (10.5 MB/s on `bench:throughput`, -4.3 µs per append on `bench:crossing`), so compare like with like. +x86_64 run on 2026-09-01 landed in that lower range (10.5 MB/s on +`bench:throughput`, 4.3 µs per append on `bench:crossing`), so compare like +with like — every figure in this document is from the arm64 build. Full configuration matrix, same run: @@ -114,35 +163,61 @@ because the preset changes which constructs md4c looks for. ### Cost by document shape +The four `bench:pathological` rows are cold medians per pipeline stage — parse, +tail repair, run segmentation, run projection — not parse times. + | Benchmark | md4c, as shipped | | --- | --- | -| Cold parse, 96.4 kB corpus, AST only (`bench:throughput`) | 4.58 ms/parse mean (min 3.56, p95 6.35) → 21.1 MB/s mean, 27.1 MB/s best-of-run, 2772 blocks | -| Nested brackets, 20 kB (`bench:pathological`) | 0.47 ms | -| Alternating emphasis openers, 104 kB | 0.40 ms | -| 32 × 500 table, 113 kB | 10.3 ms (min 7.4, max 14.3 over 3 runs) | -| Deep blockquotes (1500 levels), 3 kB | 1.85 ms cold; 0.25 ms warm (`bench:crossing`) | -| Streaming replay, 131 chunks / 1.2 kB (`bench:streaming`) | 3.1 ms of append time (3.3 ms replay total), p50 0.02 ms/chunk, p99 0.18 ms | -| Streaming replay, 1055 chunks / 9.3 kB (`--replicas 8`) | 13.1 ms of append time (30.7 ms replay total), p50 0.01 ms/chunk, p99 0.05 ms | - -Three rows need a note: +| Full-document parse, 96.4 kB corpus, AST only, warm (`bench:throughput`) | 4.35 ms/parse mean (min 3.41, p95 5.46) → 22.2 MB/s mean, 28.3 MB/s best-of-run, 2772 blocks | +| Nested brackets, 20 kB (`bench:pathological`) | parse 0.46 · repair 2.01 · segment 0.02 · project 0.02 ms | +| Alternating emphasis openers, 104 kB | parse 0.53 · repair 7.04 · segment 0.08 · project 0.07 ms | +| 32 × 500 table, 113 kB | parse 10.52 · repair 0.38 · segment 2.89 · project 7.61 ms | +| Deep blockquotes (1500 levels), 3 kB | parse 1.57 · repair 0.09 · segment 0.56 · project 2.53 ms cold; 0.22 ms warm parse (`bench:crossing`) | +| Streaming replay, 131 chunks / 1.2 kB (`bench:streaming`, re-run 2026-09-03) | 2.52–2.80 ms of append time per replay (median of 3, three runs), p50 0.02 ms/chunk, p99 0.06–0.16 ms | +| Streaming replay, 1055 chunks / 9.3 kB (`--replicas 8`, re-run 2026-09-03) | 14.98 ms of append time (34.83 ms replay total), p50 0.01 ms/chunk, p99 0.09 ms | +| Streaming replay, 2484 chunks / 21.9 kB, one 420-item bullet list (re-run 2026-09-03) | p50 0.28 ms/chunk, mean parse input 10,854 chars — never anchors; 731 and 769 ms of append time in two runs (per-replay spread 715–945 ms) | +| Projection over the replayed transcript (`bench:projection`) | 6,042 characters projected for 1,162 of document (5.2×), worst single projection 274 | + +Five notes on the table: - **The many-cell table** is the most expensive shape per byte and the - noisiest: 10.3 ms for 113 kB, about twice what the same volume of prose - costs, with a 7.4 to 14.3 ms spread across three runs. 113 kB of `| cell |` - is 16,000 cells, each a cell node with a text child. Allocating them costs - more than finding them. + noisiest: 10.5 ms to parse 113 kB, about twice what the same volume of prose + costs, and 7.6 ms to project it. 113 kB of `| cell |` is 16,000 cells, each a + cell node with a text child. Allocating them costs more than finding them. - **Deep blockquotes** produce the fattest wire buffer: 3 kB of source, 72 kB of buffer (24×), because every nesting level is its own 24-byte enter/leave - event pair. The decode share is 44%, in line with the other shapes, so + event pair. The decode share is 49%, in line with the other shapes, so crossing cost tracks node count rather than bytes. The pathological bench's - 1.85 ms for the same input is a cold run with no warmup, on purpose: a - warmed-up JIT is not what a DoS attempt meets. -- **The incremental-vs-full ratio** is 0.435 on the 1.2 kB transcript - (3.08 ms streamed vs 131 full reparses at 0.05 ms each) and 0.065 at ×8 - (13.05 ms vs 1055 × 0.19 ms), falling as the document grows. The structural - number does not move: parse input per append is mean 107 / p95 254 / max - 277 characters at both sizes, and 11 of 130 appends (88 of 1047 at ×8) + 1.57 ms parse for the same input is a cold run with no warmup, on purpose: a + warmed-up JIT is not what a DoS attempt meets. It is also the one shape whose + cost is in the projection rather than the parse. +- **The two stages that dominate** — the repair on alternating emphasis + openers, the projection on the table — were invisible while this bench timed + only the parse. md4c is linear on all four shapes, so a parse-only gate + guarded the stage that was never going to fail. +- **The incremental-vs-full ratio** now compares matched statistics: both + sides are the median, across repeats, of ONE replay's total — the streamed + side summing its appends, the naive side summing the same number of full + reparses of the final document — after a matching untimed warmup, and the + bench prints both totals and their min–max spread. Re-measured 2026-09-03 on + this machine, `--repeat 3`, three runs: 0.666 / 0.664 / 0.647 on the 1.2 kB + transcript (streamed 2.78 / 2.80 / 2.52 ms against naive 4.17 / 4.22 / + 3.90 ms over 131 chunks) and 0.098 at ×8 (14.98 ms against 153 ms over 1055 + chunks of 9.3 kB), falling as the document grows. Still quote it as a band: + each of those runs printed a spread reaching 0.447–0.913, so the two + distributions overlap at 1.2 kB and only the ×8 figure is a clean win. The + structural numbers do not move: parse input per append is mean 105 / p95 254 + / max 277 characters at both sizes, and 11 of 130 appends (88 of 1047 at ×8) skipped the engine entirely. +- **The giant-list replay** is the same measurement on a shape that never + anchors, and it inverts: mean parse input 10,854 characters of a 21,927-char + document (max 21,881, so 99.8% of the whole thing), 2389 of 2389 appends + reached the engine, and 790 nodes lost identity per chunk. Those counts are + exact; the ratio is not, and it is above 1 in every run — 1.227 and 1.466 in + two runs on 2026-09-03 (streamed 731 and 769 ms against naive 596 and + 525 ms), i.e. slower than reparsing the document from scratch on every + token. A list offers no safe anchor, so nothing settles and nothing + downstream can be reused. ### Where a native parse spends its time (`bench:crossing`) @@ -151,25 +226,29 @@ buffer into the AST, (c) the whole `engine.parse` call. | Workload | (a) native parse + encode | (b) JS decode → AST | (c) total | (b) share | Wire buffer | | --- | --: | --: | --: | --: | --- | -| Append tail, 64 B (the streaming case) | 1.59 µs | 1.15 µs | 2.68 µs | 43% | 387 B (6.05× source), 13 events | -| Chat reply, 828 B | 7.05 µs | 5.19 µs | 11.92 µs | 43% | 1.7 kB (2.07× source), 65 events | -| Spec corpus ×4, 96.4 kB | 2.01 ms | 1.69 ms | 3.69 ms | 46% | 547.7 kB (5.68× source), 21,370 events | -| Deep blockquotes, 1500 levels, 3 kB | 141 µs | 111 µs | 251 µs | 44% | 72.2 kB (24.0× source), 3,005 events | -| Many-cell table, 32 × 500, 113.4 kB | 2.12 ms | 2.29 ms | 4.35 ms | 53% | 1.18 MB (10.4× source), 49,106 events | +| Append tail, 64 B (the streaming parse input) | 1.79 µs | 1.31 µs | 3.03 µs | 43% | 387 B (6.05× source), 13 events | +| Chat reply, 828 B | 6.62 µs | 4.08 µs | 10.65 µs | 38% | 1.7 kB (2.06× source), 65 events | +| Spec corpus ×4, 96.4 kB | 1.85 ms | 1.53 ms | 3.41 ms | 45% | 532.7 kB (5.52× source), 21,378 events | +| Deep blockquotes, 1500 levels, 3 kB | 127 µs | 109 µs | 223 µs | 49% | 72.2 kB (24.0× source), 3,005 events | +| Many-cell table, 32 × 500, 113.4 kB | 2.08 ms | 2.10 ms | 4.14 ms | 51% | 1.18 MB (10.4× source), 49,106 events | The last two rows are the pathological bench's shapes, built by the same generators in `bench/support.mjs`, timed with pinned iteration counts. The -bench prints its timer floor, 66 ns per timed region on this machine, and -never subtracts it; on the 64 B row that is ~4% of a stage. +bench prints its timer floor, 76 ns per timed region on this machine, and +never subtracts it; on the 64 B row that is 4–6% of a stage. Two takeaways: -- A streaming append costs ~2.7 µs end to end, ~0.02% of a 16.7 ms frame. The - remaining on-device cost is React and text layout, not parsing (speed - metrics 5 and 6, both planned). -- The decode is 43–53% of a parse by shape, and it is the half written in - JavaScript. The decoder reads the wire buffer at 324–337 MB/s on prose and - 514–649 MB/s on the node-dense shapes. Lazy per-block AST materialization +- `engine.parse` on a 64 B tail costs ~3.0 µs, ~0.02% of a 16.7 ms frame. That + is the crossing and nothing else: a whole `StreamSession.append` — anchor + scan, tail repair, parse, span shift, snapshot — averages 14 µs over the + 1055-chunk ×8 replay and 19–21 µs over the 131-chunk one (the streamed total + the streaming bench prints, divided by its chunk count, 2026-09-03), and the + mean parse input there is 105 characters rather than 64 B. The remaining on-device cost + is React and text layout, not parsing (speed metrics 5 and 6, both planned). +- The decode is 38–51% of a parse by shape, and it is the half written in + JavaScript. The decoder reads the wire buffer at 297–418 MB/s on prose and + 562–660 MB/s on the node-dense shapes. Lazy per-block AST materialization was considered as the next win and rejected (2026-08-25): the span-widening pipeline that eager block spans need is ~60% of decode, and renderers read every block, so lazy children would re-pay it (about 1.6× total) or need a @@ -179,11 +258,13 @@ Two takeaways: | Command | Result | | --- | --- | -| `npx jest` / `npx tsc --noEmit` | 31 suites, 792 tests, all green; typecheck clean | +| `npx jest` / `npx tsc --noEmit` | 46 suites, 1561 tests, all green; typecheck clean. Nothing asserts those two counts, so read them as of 2026-09-03 | | `npm run conformance` | 651/652 (99.85%) on CommonMark 0.31.2, 0 examples threw. Every section at 100% except HTML blocks (43/44); the single failure is example 174, `>
\n> foo\n\nbar`, where md4c ends the quoted HTML block differently from cmark. Per-section table: `conformance/report-native.json` | -| `npx jest src/engine/native` | 6 suites, 164 tests: ABI parity against the C++ headers, host-binding resolution, named-construct documents, span invariants over the whole spec corpus, `underline`, smart punctuation | -| `npm run bench:pathological -- --budget 2000` | all four adversarial cases pass, the slowest at ~10 ms against a 2000 ms budget | -| `npm run bench:streaming` | parse input per append stays flat at mean 107 / p95 254 / max 277 chars regardless of stream length; 11 of 130 appends (88 of 1047 at `--replicas 8`) skipped the engine entirely | +| `npx jest src/engine/native` | 6 suites, 211 tests: ABI parity against the C++ headers, host-binding resolution, named-construct documents, span invariants over the whole spec corpus, `underline`, smart punctuation | +| `npm run bench:pathological -- --require-engine --budget-parse 750 --budget-repair 750 --budget-segment 200 --budget-project 750` | all four adversarial cases pass in all four stages, the slowest at 10.5 ms (the table's parse) against its 750 ms budget and `segment` at 2.9 ms against 200 ms; the repair-scaling section reports 0.43× cost-per-bracket growth from n=500 to n=8000, well under its 2.00× limit. This is the step CI runs, and `--require-engine` is what stops it passing having measured nothing | +| `npm run bench:streaming` | on the sprint-review transcript, parse input per append stays flat at mean 105 / p95 254 / max 277 chars regardless of stream length, and 11 of 130 appends (88 of 1047 at `--replicas 8`) skipped the engine entirely; on the giant-list transcript nothing anchors, mean parse input is 10,854 of 21,927 chars and 0 appends skip the engine | +| `npm run bench:streaming -- --transcript conformance/fixtures/transcript-giant-list.json --require-engine --repeat 1 --budget-chunk 20 --budget-finalize 50` | gates: p99 append latency and finalize time on the never-anchoring transcript, the shape where every append re-reads the accumulated text. Measured on 2026-09-03: p99 0.86 ms against the 20 ms budget and finalize 0.59 ms against 50 ms (0.75 and 0.85 ms in a second run). ci.yml's test job and release.yml both run this exact step; `--require-engine` stops it passing having measured nothing, and a budgeted run that timed no chunk exits 1 | +| `npm run bench:projection -- --require-engine` | gates: projected characters per document character stay flat as the document doubles — 0.99× growth cached against 1.98× uncached, 5.2× amplification at either size against 32.6× and 64.6×. Cached growth above 1.25× per doubling exits 1, and ci.yml and release.yml run it. On the giant-list transcript both pipelines measure ~553× (growth 1.00× against 2.09×), because a list never anchors and nothing settles to reuse | ## Offset map and decoder pass (2026-08-25) @@ -212,13 +293,14 @@ decoder throughput are in the crossing table above. | # | Metric | Status | Where | What it reports | | --- | --- | --- | --- | --- | -| 1 | Cold-parse throughput | implemented | `bench/throughput.mjs` | MB/s mean and best-of-run over the spec corpus plus fixtures, replicated by `--replicas`, timed through `parseDocument`. Block count printed as a sanity check (2772 for ×4). Planned: per-corpus breakdown. | -| 2 | Pathological-input budget | implemented; gates with `--budget` | `bench/pathological.mjs` | Median cold time per adversarial case: 10k nested brackets, nested emphasis runs, 1500 nested blockquotes, a 32 × 500 table. With `--budget MS`, exceeding it or crashing exits 1. No warmup, so numbers run higher than the same input warm. | -| 3 | Streaming replay | implemented | `bench/streaming-replay.mjs` | ms/chunk at p50/p95/p99 and blocks whose identity changed per chunk (p50 7, p95 21, max 31, mean 8.5, identical at ×1 and ×8; churn is in the unsettled tail). Fixture: `conformance/fixtures/transcript-sprint-review.json`, hand-built deltas of 1–18 UTF-16 units with surrogate-safe splits. `--transcript PATH` replays any transcript of the same shape. | -| 4 | Incremental-vs-full reparse ratio | implemented | reported by `bench/streaming-replay.mjs` | Total streamed append time over `chunks × full reparse of the final document`, same engine both sides, plus parse-input size per append and the count of construct-free fast-path appends. Lower is better; falls as documents grow. Do not quote `--quick`. A stream that never anchors (one huge list, one giant paragraph, an unclosed fence) reparses its whole tail every chunk. | +| 1 | Full-document parse throughput (warm) | implemented | `bench/throughput.mjs` | MB/s mean and best-of-run over the spec corpus plus fixtures, replicated by `--replicas`, timed through `parseDocument` after 3 untimed warmups — warm, not cold; "cold" in this document means the no-warmup `bench:pathological`, and the deep-blockquote row shows the two differing ~7×. Block count printed as a sanity check (2772 for ×4). Planned: per-corpus breakdown. | +| 2 | Pathological-input budget | implemented; gates with `--budget`, and CI runs it | `bench/pathological.mjs` | Median cold time per adversarial case, per pipeline stage: parse (md4c + decode), repair (`repairTail` over the whole input, which is what a stream that never anchors hands it), segment (`segmentRuns`), project (`projectRun` over every run). Cases: 10k nested brackets, alternating emphasis openers, 1500 nested blockquotes, a 32 × 500 table. md4c is linear on all four, so a parse-only gate guarded the one stage that was never going to fail. `--budget MS` applies per stage, and `--budget-parse`, `--budget-repair`, `--budget-segment` and `--budget-project` override it for one stage (the global stays the default for the others) — which is how CI runs it, because a single budget loose enough for the table's parse leaves `segment`, four orders of magnitude cheaper, free to get a hundred times slower and still pass. Exceeding a budget, or throwing in any stage, exits 1 — a stack overflow on adversarial input is a denial of service whatever its runtime — and so does a gated run that produced no samples at all (`--runs 0`, or `--require-engine` meeting an unresolvable addon), because a gate over nothing is a failure, not a pass. No warmup, so numbers run higher than the same input warm. A second section, `repair scaling`, runs `repairTail` over `'x [ '.repeat(n)` for n in {500,1000,2000,4000,8000} against a same-length linear control and gates the growth in cost-per-bracket (flat ~1.00× is linear, ~16× would be quadratic, over 2.00× fails) — the four fixed-size cases cannot tell a slow linear pass from a fast quadratic one. 1500 is a budget case, not a limit: the JS pipeline is depth-safe at any depth (pinned at 20,000 levels across `src/stream/`, `src/selection/` and `src/view/`), and the only bound is `MAX_RENDER_DEPTH = 64` in `src/view/renderers.tsx`. | +| 3 | Streaming replay | implemented; gates with `--budget`, and CI runs it | `bench/streaming-replay.mjs` | ms/chunk at p50/p95/p99 and nodes (blocks and inlines) whose identity changed per chunk — the label the bench prints, and the count that predicts React re-render cost. On `transcript-sprint-review.json`: p50 7, p95 21, max 31, mean 8.3, identical at ×1 and ×8; churn is in the unsettled tail. On `transcript-giant-list.json`: mean 790. Both are replayed by default, in hand-built deltas of 1–18 UTF-16 units with surrogate-safe splits; `--transcript PATH` narrows to one, or replays any transcript of the same shape. What the budgets gate, per transcript, is the p99 append latency (`--budget-chunk`) and the single clean parse `finalize` does (`--budget-finalize`), with `--budget` as the default for both; ci.yml and release.yml run it over `transcript-giant-list.json`, the never-anchoring shape where every append re-reads the accumulated text (2484 chunks, 2389 of 2389 appends reaching the engine, mean parse input 10,854 chars). That is where a repair or splice pass that stops being linear shows up first, and it shows up in the number a user feels as jank. Without a budget nothing fails; a budgeted run that timed nothing exits 1. | +| 4 | Incremental-vs-full reparse ratio | implemented | reported by `bench/streaming-replay.mjs` | The median of one replay's streamed total against the median of one replay's naive total — the same number of full reparses of the final document, same engine, after a matching untimed warmup — plus both totals, their min–max spread, parse-input size per append and the count of construct-free fast-path appends. Lower is better; falls as documents grow. Both sides are now the same statistic (it used to divide a sum of chunk times by a median full parse × the chunk count), but the two distributions still overlap at 1.2 kB, so quote it as a band rather than a number and read the spread line with it. Do not quote `--quick`. A stream that never anchors reparses its whole tail every chunk: one huge list, one giant paragraph, an unclosed fence, or an unterminated HTML block of CommonMark type 1–5 (``, +`?>`, a declaration's `>`) cannot grow, so it anchors like anything +else — the same argument `closed: true` makes for a fenced code block. A +literal matching no start condition is one md4c did not start as type 1-4, so +a blank line really does end it and it anchors as types 6 and 7 do. +Keying on the opener alone cost every raw-HTML stream its anchor entirely: a +document of one-line `` blocks reparsed from offset 0 on every +append. None of the `htmlBlock` cases is reachable under the default +`html: 'strip'`, which emits no `htmlBlock` node at all. Every excluded block +freezes once an anchor-safe block after it is followed by a blank line. ### Tail-only reparse ``` -tailInput = repairTail(source.slice(anchor), cleanSeed, options).text -document = frozenPrefixBlocks ++ shiftSpans(parse(tailInput).blocks, anchor) +repaired = repairTail(source.slice(anchor), cleanSeed, options, repair, carry) +document = frozenPrefixBlocks ++ shiftSpans(parse(repaired.text).blocks, anchor) ``` `shiftSpans` (`src/stream/shiftSpans.ts`) deep-clones each tail block with -spans rebased by `anchor`; engine output is never mutated. Frozen prefix +spans rebased by `anchor`; engine output is never mutated. The clone walks +an explicit job stack rather than recursing, as do +`trimTrailingPlaceholders` and finalize's structure check, so nesting depth +costs heap rather than stack: a 40 kB `> ` prefix streams, freezes and +finalizes instead of throwing `RangeError` out of an append. Frozen prefix blocks are spliced back by reference, so settled identity holds by construction. With no anchor yet (one giant list, a single huge paragraph, an -unclosed fence), the tail is the whole source and the update is a correct -full reparse. +unclosed fence), the tail is the whole source, so the update is a correct +full reparse plus a full repair pass over the same text. The parse is +unavoidable; the repair's inline scan resumes from the previous append +(`carry`, below) instead of restarting, so only the new delta is scanned and +the pass measures about a third of the md4c parse beside it on a 32 kB +paragraph streamed in 18-character deltas. ### The construct-free fast path @@ -149,56 +322,178 @@ and `value += delta`, when: onto the raw source and reaches the end of the text; - the previous repair made zero changes; - the delta does not end in a space, a tab, or a lone high surrogate; -- the last line does not end in a bare-autolink candidate (`https:`, `www.`), - an HTML-block opener stub (`<`, ``/`
`/`