diff --git a/changelog.d/1963-cut-wellformed.fixed.md b/changelog.d/1963-cut-wellformed.fixed.md new file mode 100644 index 000000000..05ecacd95 --- /dev/null +++ b/changelog.d/1963-cut-wellformed.fixed.md @@ -0,0 +1 @@ +A text bound no longer keeps an unpaired surrogate the caller handed in ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)): every character bound in the host goes through the shared `cut`, and that helper moved its cut point off a pair it was about to split but copied a string that already carried a lone surrogate straight through, so a card title, a note or a name that arrived broken was written to `state.json` as the same broken value โ€” read back as U+FFFD by any encoder. `cut` now flattens its input with `toWellFormed()` before it bounds, so the stored text is well formed; U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its "was cut" flag. diff --git a/core/client.js b/core/client.js index 8fa6c6a86..c114ae4de 100644 --- a/core/client.js +++ b/core/client.js @@ -1723,7 +1723,10 @@ var isHighSurrogate = (code) => code >= 55296 && code <= 56319; var isLowSurrogate = (code) => code >= 56320 && code <= 57343; var codePointStart = (text2, at) => at > 0 && isLowSurrogate(text2.charCodeAt(at)) && isHighSurrogate(text2.charCodeAt(at - 1)) ? at - 1 : at; var codePointEnd = (text2, end) => end > 0 && isHighSurrogate(text2.charCodeAt(end - 1)) && isLowSurrogate(text2.charCodeAt(end)) ? end - 1 : end; -var cut = (text2, max) => ({ text: text2.slice(0, codePointEnd(text2, max)), cut: text2.length > max }); +var cut = (text2, max) => { + const flat = String(text2).toWellFormed(); + return { text: flat.slice(0, codePointEnd(flat, max)), cut: flat.length > max }; +}; // src/shared/remote-label.js function cleanPeerName(name, fallback = "") { diff --git a/core/src/shared/text-boundary.js b/core/src/shared/text-boundary.js index af6579e24..dd411bc85 100644 --- a/core/src/shared/text-boundary.js +++ b/core/src/shared/text-boundary.js @@ -13,4 +13,10 @@ export const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charC export const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) // A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped. -export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max }) +// The input is flattened first (#1963): a string that already carried an unpaired surrogate went +// straight through, so the caller stored a value that was not well formed. U+FFFD is one UTF-16 unit +// for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag. +export const cut = (text, max) => { + const flat = String(text).toWellFormed() + return { text: flat.slice(0, codePointEnd(flat, max)), cut: flat.length > max } +} diff --git a/core/test/text-boundary.test.js b/core/test/text-boundary.test.js new file mode 100644 index 000000000..32ed30acd --- /dev/null +++ b/core/test/text-boundary.test.js @@ -0,0 +1,58 @@ +// #1963: `cut` is the last bound before card text reaches `state.json`, and it only moved its own +// cut point off a pair it was about to split. A string that arrived already carrying an unpaired +// high or low surrogate was copied straight through, so the stored value was not well formed โ€” a +// UTF-8 encoder writes that unit as U+FFFD, and reading the card back gave a different string than +// the one the caller stored. Every other normaliser in the host flattens its input before bounding +// it (`delivery.js`, `remote-label.js`, `room-share.js`), and this bound does the same now. +// U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix +// and its `cut` flag. +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { cut } from '../src/shared/text-boundary.js' + +const LONE_HIGH = 'a\uD800b' +const LONE_LOW = 'a\uDC00b' + +test('a lone high surrogate in the input comes back as U+FFFD', () => { + const out = cut(LONE_HIGH, 8) + assert.equal(out.text, 'a\uFFFDb') + assert.equal(out.text.isWellFormed(), true) + // One unit in, one unit out: repairing never spends part of the budget or grows the text. + assert.equal(out.text.length, 3) + assert.equal(out.cut, false) +}) + +test('a lone low surrogate in the input comes back as U+FFFD', () => { + const out = cut(LONE_LOW, 8) + assert.equal(out.text, 'a\uFFFDb') + assert.equal(out.text.isWellFormed(), true) + assert.equal(out.cut, false) +}) + +test('a lone surrogate is repaired before the bound, not after it', () => { + // Five units, max four: the prefix that is kept still carries the unit the repair replaced. + const out = cut('ab\uD800cd', 4) + assert.equal(out.text, 'ab\uFFFDc') + assert.equal(out.text.isWellFormed(), true) + assert.equal(out.cut, true) +}) + +test('well-formed input keeps its exact prefix and flag', () => { + const cases = [ + ['', 5, '', false], + ['abcdef', 3, 'abc', true], + ['abc', 3, 'abc', false], + ['abc', 0, '', true], + ['a๐Ÿ˜€', 3, 'a๐Ÿ˜€', false], + // The pair straddles the bound, so it is dropped whole instead of being cut in half. + ['aa๐Ÿ˜€', 3, 'aa', true], + ['๐Ÿ‘จ\u200D๐Ÿ‘ฉ\u200D๐Ÿ‘ง', 4, '๐Ÿ‘จ\u200D', true], + ['a'.repeat(2000), 2000, 'a'.repeat(2000), false], + ] + for (const [text, max, expected, flag] of cases) { + const out = cut(text, max) + assert.deepEqual(out, { text: expected, cut: flag }, `${JSON.stringify(text)} @ ${max}`) + assert.equal(out.text.isWellFormed(), true) + } +}) diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 2c18c1dd7..1b671490f 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | --- | --- | | `core/index.js` | Server entry. | | `core/src/host/` | Host-side helpers (non-hooks). | -| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit โ€” one character costs one whatever its UTF-16 width โ€” and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it โ€” the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` โ€” so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it โ€” the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) โ€” each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `โ€ฆ (cut)` instead of `โ€ฆ (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` โ€” `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) โ€” so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="โ€ฆ"` attribute โ€” so none of them can end in half a surrogate pair. | +| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit โ€” one character costs one whatever its UTF-16 width โ€” and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it โ€” the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` โ€” so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it โ€” the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) โ€” each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `โ€ฆ (cut)` instead of `โ€ฆ (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` โ€” `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) โ€” so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="โ€ฆ"` attribute โ€” so none of them can end in half a surrogate pair. Since #1963 `cut` also flattens its input before it bounds โ€” `String(text).toWellFormed()` first, `codePointEnd` on the flat string โ€” so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). | | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative โ€” an addressed Bot answers as before. | | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. | | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `'s Chief` / ` ยท 's Bot`, ` ยท `, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). | diff --git a/patches/ds-bot/0008-client-bundle.patch b/patches/ds-bot/0008-client-bundle.patch index 59196ba9f..c0db79a19 100644 --- a/patches/ds-bot/0008-client-bundle.patch +++ b/patches/ds-bot/0008-client-bundle.patch @@ -1104,7 +1104,7 @@ diff --git a/client.js b/client.js if (!REFUSALS.has(code)) { noteClientError(endpoint, Object.assign(new Error(result.error?.message ?? "Request failed"), { code })); markNoted(error); -@@ -1097,6 +1097,396 @@ var call = async (endpoint, payload, signal) => { +@@ -1097,6 +1097,399 @@ var call = async (endpoint, payload, signal) => { return result.value; }; @@ -1113,7 +1113,10 @@ diff --git a/client.js b/client.js +var isLowSurrogate = (code) => code >= 56320 && code <= 57343; +var codePointStart = (text2, at) => at > 0 && isLowSurrogate(text2.charCodeAt(at)) && isHighSurrogate(text2.charCodeAt(at - 1)) ? at - 1 : at; +var codePointEnd = (text2, end) => end > 0 && isHighSurrogate(text2.charCodeAt(end - 1)) && isLowSurrogate(text2.charCodeAt(end)) ? end - 1 : end; -+var cut = (text2, max) => ({ text: text2.slice(0, codePointEnd(text2, max)), cut: text2.length > max }); ++var cut = (text2, max) => { ++ const flat = String(text2).toWellFormed(); ++ return { text: flat.slice(0, codePointEnd(flat, max)), cut: flat.length > max }; ++}; + +// src/shared/remote-label.js +function cleanPeerName(name, fallback = "") { @@ -1841,7 +1844,6 @@ diff --git a/client.js b/client.js return spans; } - --// src/client/text.js +function unmatchedMentions(text2, bots, { references = [] } = {}) { + const value = String(text2 ?? ""); + const skip = [ @@ -1890,7 +1892,7 @@ diff --git a/client.js b/client.js + return stripFence(out).replace(/\s+/g, " ").trim(); +} + -+// src/client/text.js + // src/client/text.js var MENTION_ORIGIN = "https://bots.local/"; -function linkMentions(text, bots, selfId) { - if (!text || !text.includes("@")) return text; @@ -2131,7 +2133,7 @@ diff --git a/client.js b/client.js var formatTokens = (count) => count >= 1e6 ? `${(count / 1e6).toFixed(1)}M` : count >= 1e3 ? `${(count / 1e3).toFixed(1)}K` : String(count); function usageOf(list, ids) { let input = 0; -@@ -1624,902 +1624,1982 @@ function usageOf(list, ids) { +@@ -1624,901 +1624,1981 @@ function usageOf(list, ids) { const hit = input > 0 ? Math.round(cacheRead / input * 100) : 0; return t("{tokens} tok ยท Cache hit {hit}%", { tokens: formatTokens(input + output), hit }); } @@ -2395,11 +2397,15 @@ diff --git a/client.js b/client.js - }; - animation.oncancel = () => { - if (!node.getAnimations().some((other) => other.id === "bt-morph" && other !== animation)) delete node.dataset.morph; +- }; +- }); +function isAgentSession(roster, cache, sessionId) { + if (sessionId === void 0 || sessionId === null || sessionId === "") return false; + if (roster.ready) return roster.agentOf[sessionId] !== void 0; + return cache.has(sessionId); -+} + } +-function useLinger(value, ms) { +- const [kept, setKept] = (0, import_react.useState)(value); +function countCards(state, sessionId, callIds, delta) { + const own = { ...state[sessionId] }; + for (const callId of callIds) { @@ -2416,7 +2422,9 @@ diff --git a/client.js b/client.js +var botState = (running, activity, alert) => running ? STATES.includes(activity) ? activity : "working" : alert ? "alert" : "idle"; +var toolState = (name) => /^(message_bot|stop_bot|create_bot|update_bot|create_group|update_group|delete_group|post_to_group)$/.test(name) ? "sending" : /search|fetch|browse|read|grep|glob|list|recall/i.test(name) ? "searching" : "working"; +function useGaze(ref, enabled) { -+ (0, import_react.useEffect)(() => { + (0, import_react.useEffect)(() => { +- if (value) { +- setKept(value); + if (!enabled) return void 0; + let frame = 0; + let point = null; @@ -2428,8 +2436,7 @@ diff --git a/client.js b/client.js + const clamp = (value) => Math.max(-1, Math.min(1, value)); + node.style.setProperty("--bt-gx", clamp((point.x - box.left - box.width / 2) / 160).toFixed(2)); + node.style.setProperty("--bt-gy", clamp((point.y - box.top - box.height / 2) / 160).toFixed(2)); - }; -- }); ++ }; + const onMove = (event) => { + point = { x: event.clientX, y: event.clientY }; + if (!frame) frame = requestAnimationFrame(place); @@ -2447,9 +2454,7 @@ diff --git a/client.js b/client.js + document.documentElement.removeEventListener("pointerleave", onLeave); + }; + }, [enabled]); - } --function useLinger(value, ms) { -- const [kept, setKept] = (0, import_react.useState)(value); ++} +var FIDGETS = ["hop", "sway", "stretch", "look"]; +function BotMark({ bot, look: given, size = 36, state = "idle", live = false, gaze = false, pokeable = false, label }) { + const look = given ?? (bot?.remote ? lookAvoidingLocals(bot) : lookOf(bot)); @@ -2529,9 +2534,7 @@ diff --git a/client.js b/client.js +function useSettled(state) { + const prev = (0, import_react.useRef)(state); + const [settled, setSettled] = (0, import_react.useState)(false); - (0, import_react.useEffect)(() => { -- if (value) { -- setKept(value); ++ (0, import_react.useEffect)(() => { + const was = prev.current; + prev.current = state; + if (state !== "idle") { @@ -3976,7 +3979,6 @@ diff --git a/client.js b/client.js -.bt-mem-summary>:nth-child(5){animation-delay:.12s} -.bt-mem-summary>:nth-child(6){animation-delay:.15s} -.bt-mem-summary>:nth-child(n+7){animation-delay:.18s} --@keyframes bt-mem-line{from{opacity:0;transform:translateY(4px)}} +// src/client/motion.js +var import_react4 = require("react"); +var reducedMotion = () => window.matchMedia?.("(prefers-reduced-motion: reduce)").matches === true; @@ -4978,10 +4980,9 @@ diff --git a/client.js b/client.js +.bt-mem-summary>:nth-child(5){animation-delay:.12s} +.bt-mem-summary>:nth-child(6){animation-delay:.15s} +.bt-mem-summary>:nth-child(n+7){animation-delay:.18s} -+@keyframes bt-mem-line{from{opacity:0;transform:translateY(4px)}} + @keyframes bt-mem-line{from{opacity:0;transform:translateY(4px)}} .bt-mem-skel{display:flex;flex-direction:column;gap:24px} .bt-mem-skel-block{display:flex;flex-direction:column;gap:10px} - .bt-mem-skel-block span{display:block;height:12px;border-radius:6px;background:linear-gradient(90deg,var(--bt-hover) 25%,var(--bt-active) 50%,var(--bt-hover) 75%);background-size:200% 100%;animation:bt-mem-skel 1.4s ease-in-out infinite} @@ -2574,2206 +2574,4679 @@ body[data-ds-dark-theme] .bt-theme-chip i{background:var(--da)} @media (max-width:640px){.bt-mem{width:100vw;height:100%;border-radius:0;border:0}.bt-mem-body{padding:18px 18px 20px}.bt-mem-foot{padding:8px 12px 12px}} @media (prefers-reduced-motion:reduce){.bt-mem,.bt-mem-scrim,.bt-mem-summary>*,.bt-mem-empty,.bt-mem-reply,.bt-mem-skel-block span,.bt-mem-entries,.bt-mem-send>*,.bt-mem-changes li svg{animation:none}} diff --git a/patches/ds-bot/files/src/shared/text-boundary.js b/patches/ds-bot/files/src/shared/text-boundary.js index af6579e24..dd411bc85 100644 --- a/patches/ds-bot/files/src/shared/text-boundary.js +++ b/patches/ds-bot/files/src/shared/text-boundary.js @@ -13,4 +13,10 @@ export const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charC export const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end) // A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped. -export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max }) +// The input is flattened first (#1963): a string that already carried an unpaired surrogate went +// straight through, so the caller stored a value that was not well formed. U+FFFD is one UTF-16 unit +// for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag. +export const cut = (text, max) => { + const flat = String(text).toWellFormed() + return { text: flat.slice(0, codePointEnd(flat, max)), cut: flat.length > max } +} diff --git a/patches/ds-bot/files/test/text-boundary.test.js b/patches/ds-bot/files/test/text-boundary.test.js new file mode 100644 index 000000000..32ed30acd --- /dev/null +++ b/patches/ds-bot/files/test/text-boundary.test.js @@ -0,0 +1,58 @@ +// #1963: `cut` is the last bound before card text reaches `state.json`, and it only moved its own +// cut point off a pair it was about to split. A string that arrived already carrying an unpaired +// high or low surrogate was copied straight through, so the stored value was not well formed โ€” a +// UTF-8 encoder writes that unit as U+FFFD, and reading the card back gave a different string than +// the one the caller stored. Every other normaliser in the host flattens its input before bounding +// it (`delivery.js`, `remote-label.js`, `room-share.js`), and this bound does the same now. +// U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix +// and its `cut` flag. +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { cut } from '../src/shared/text-boundary.js' + +const LONE_HIGH = 'a\uD800b' +const LONE_LOW = 'a\uDC00b' + +test('a lone high surrogate in the input comes back as U+FFFD', () => { + const out = cut(LONE_HIGH, 8) + assert.equal(out.text, 'a\uFFFDb') + assert.equal(out.text.isWellFormed(), true) + // One unit in, one unit out: repairing never spends part of the budget or grows the text. + assert.equal(out.text.length, 3) + assert.equal(out.cut, false) +}) + +test('a lone low surrogate in the input comes back as U+FFFD', () => { + const out = cut(LONE_LOW, 8) + assert.equal(out.text, 'a\uFFFDb') + assert.equal(out.text.isWellFormed(), true) + assert.equal(out.cut, false) +}) + +test('a lone surrogate is repaired before the bound, not after it', () => { + // Five units, max four: the prefix that is kept still carries the unit the repair replaced. + const out = cut('ab\uD800cd', 4) + assert.equal(out.text, 'ab\uFFFDc') + assert.equal(out.text.isWellFormed(), true) + assert.equal(out.cut, true) +}) + +test('well-formed input keeps its exact prefix and flag', () => { + const cases = [ + ['', 5, '', false], + ['abcdef', 3, 'abc', true], + ['abc', 3, 'abc', false], + ['abc', 0, '', true], + ['a๐Ÿ˜€', 3, 'a๐Ÿ˜€', false], + // The pair straddles the bound, so it is dropped whole instead of being cut in half. + ['aa๐Ÿ˜€', 3, 'aa', true], + ['๐Ÿ‘จ\u200D๐Ÿ‘ฉ\u200D๐Ÿ‘ง', 4, '๐Ÿ‘จ\u200D', true], + ['a'.repeat(2000), 2000, 'a'.repeat(2000), false], + ] + for (const [text, max, expected, flag] of cases) { + const out = cut(text, max) + assert.deepEqual(out, { text: expected, cut: flag }, `${JSON.stringify(text)} @ ${max}`) + assert.equal(out.text.isWellFormed(), true) + } +})