core: cut() stores a lone surrogate the input already carried

changelog.d/1963-cut-wellformed.fixed.md

@@ -0,0 +1 @@
+A text bound no longer keeps an unpaired surrogate the caller handed in ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)): every character bound in the host goes through the shared `cut`, and that helper moved its cut point off a pair it was about to split but copied a string that already carried a lone surrogate straight through, so a card title, a note or a name that arrived broken was written to `state.json` as the same broken value — read back as U+FFFD by any encoder. `cut` now flattens its input with `toWellFormed()` before it bounds, so the stored text is well formed; U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its "was cut" flag.

core/client.js

@@ -1723,7 +1723,10 @@ var isHighSurrogate = (code) => code >= 55296 && code <= 56319;
 var isLowSurrogate = (code) => code >= 56320 && code <= 57343;
 var codePointStart = (text2, at) => at > 0 && isLowSurrogate(text2.charCodeAt(at)) && isHighSurrogate(text2.charCodeAt(at - 1)) ? at - 1 : at;
 var codePointEnd = (text2, end) => end > 0 && isHighSurrogate(text2.charCodeAt(end - 1)) && isLowSurrogate(text2.charCodeAt(end)) ? end - 1 : end;
-var cut = (text2, max) => ({ text: text2.slice(0, codePointEnd(text2, max)), cut: text2.length > max });
+var cut = (text2, max) => {
+  const flat = String(text2).toWellFormed();
+  return { text: flat.slice(0, codePointEnd(flat, max)), cut: flat.length > max };
+};
 
 // src/shared/remote-label.js
 function cleanPeerName(name, fallback = "") {

core/src/shared/text-boundary.js

@@ -13,4 +13,10 @@ export const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charC
 export const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end)
 
 // A bounded prefix that never splits a surrogate pair. `cut` says whether anything was dropped.
-export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max })
+// The input is flattened first (#1963): a string that already carried an unpaired surrogate went
+// straight through, so the caller stored a value that was not well formed. U+FFFD is one UTF-16 unit
+// for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag.
+export const cut = (text, max) => {
+  const flat = String(text).toWellFormed()
+  return { text: flat.slice(0, codePointEnd(flat, max)), cut: flat.length > max }
+}

core/test/text-boundary.test.js

@@ -0,0 +1,58 @@
+// #1963: `cut` is the last bound before card text reaches `state.json`, and it only moved its own
+// cut point off a pair it was about to split. A string that arrived already carrying an unpaired
+// high or low surrogate was copied straight through, so the stored value was not well formed — a
+// UTF-8 encoder writes that unit as U+FFFD, and reading the card back gave a different string than
+// the one the caller stored. Every other normaliser in the host flattens its input before bounding
+// it (`delivery.js`, `remote-label.js`, `room-share.js`), and this bound does the same now.
+// U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix
+// and its `cut` flag.
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { cut } from '../src/shared/text-boundary.js'
+
+const LONE_HIGH = 'a\uD800b'
+const LONE_LOW = 'a\uDC00b'
+
+test('a lone high surrogate in the input comes back as U+FFFD', () => {
+  const out = cut(LONE_HIGH, 8)
+  assert.equal(out.text, 'a\uFFFDb')
+  assert.equal(out.text.isWellFormed(), true)
+  // One unit in, one unit out: repairing never spends part of the budget or grows the text.
+  assert.equal(out.text.length, 3)
+  assert.equal(out.cut, false)
+})
+
+test('a lone low surrogate in the input comes back as U+FFFD', () => {
+  const out = cut(LONE_LOW, 8)
+  assert.equal(out.text, 'a\uFFFDb')
+  assert.equal(out.text.isWellFormed(), true)
+  assert.equal(out.cut, false)
+})
+
+test('a lone surrogate is repaired before the bound, not after it', () => {
+  // Five units, max four: the prefix that is kept still carries the unit the repair replaced.
+  const out = cut('ab\uD800cd', 4)
+  assert.equal(out.text, 'ab\uFFFDc')
+  assert.equal(out.text.isWellFormed(), true)
+  assert.equal(out.cut, true)
+})
+
+test('well-formed input keeps its exact prefix and flag', () => {
+  const cases = [
+    ['', 5, '', false],
+    ['abcdef', 3, 'abc', true],
+    ['abc', 3, 'abc', false],
+    ['abc', 0, '', true],
+    ['a😀', 3, 'a😀', false],
+    // The pair straddles the bound, so it is dropped whole instead of being cut in half.
+    ['aa😀', 3, 'aa', true],
+    ['👨\u200D👩\u200D👧', 4, '👨\u200D', true],
+    ['a'.repeat(2000), 2000, 'a'.repeat(2000), false],
+  ]
+  for (const [text, max, expected, flag] of cases) {
+    const out = cut(text, max)
+    assert.deepEqual(out, { text: expected, cut: flag }, `${JSON.stringify(text)} @ ${max}`)
+    assert.equal(out.text.isWellFormed(), true)
+  }
+})

docs/subsystems/core-host.md

@@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core
 | --- | --- |
 | `core/index.js` | Server entry. |
 | `core/src/host/` | Host-side helpers (non-hooks). |
-| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/<date>-<slug>.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `… (cut)` instead of `… (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` — `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) — so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="…"` attribute — so none of them can end in half a surrogate pair. |
+| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/<date>-<slug>.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `… (cut)` instead of `… (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` — `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) — so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="…"` attribute — so none of them can end in half a surrogate pair. Since #1963 `cut` also flattens its input before it bounds — `String(text).toWellFormed()` first, `codePointEnd` on the flat string — so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). |
 | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative — an addressed Bot answers as before. |
 | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. |
 | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `<owner>'s Chief` / `<name> · <owner>'s Bot`, `<name> · <owner>`, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). |