core-host: count a Project memory entry in code points, not UTF-16 units

changelog.d/1886-memory-code-points.fixed.md

@@ -0,0 +1 @@
+The Project memory box now counts its 600-character bound in characters rather than UTF-16 units, so a note of 600 emoji — 1,200 units — is saved instead of refused as *A Project memory entry is at most 600 characters*, and 601 characters is still refused in both writers ([#1886](https://github.com/ThinkFlowLab/FlyBot/issues/1886)).

core/src/host/constants.js

@@ -23,6 +23,9 @@ export const ROUTE = '/api/bot'
 export const MAX_HOPS = 6
 // The bound on one Project memory entry (#1847): one value, shared by the assistant writer and the
 // store's `appendProjectMemory`, so no writer can store a blank entry or silently cut a long one.
+// It counts code points (#1886), not UTF-16 units: `String#length` charges an emoji two, so a 350
+// character note was refused as "over 600 characters". Count with `[...text].length`, as
+// `chat-index.js` and `busy-ack.js` already do, and keep both writers on that same unit.
 export const MAX_ENTRY = 600
 // Bot requests a Bot's own chat may hold unread; past this the Bot is most likely
 // stuck, and more messages only pile up behind it.

core/src/host/project-assistant.js

@@ -188,7 +188,9 @@ export function install(rt) {
     if (!MEMORY_KINDS.includes(kind)) throw refusal(`Unknown Project memory kind ${kind}`)
     const entry = text(body.text)
     if (entry === '') throw refusal('A Project memory entry needs text')
-    if (entry.length > MAX_ENTRY) throw refusal(`A Project memory entry is at most ${MAX_ENTRY} characters`)
+    // #1886: count code points, the same unit `appendProjectMemory` counts, so the two writers
+    // cannot disagree about which note fits (`String#length` charged an emoji two units).
+    if ([...entry].length > MAX_ENTRY) throw refusal(`A Project memory entry is at most ${MAX_ENTRY} characters`)
     const ownerId = text(body.ownerId) || ownersOf(project)[0] || ''
     if (text(body.ownerId) !== '' && !ownersOf(project).includes(text(body.ownerId))) throw outsideBoundary(project, body.ownerId)
     rt.appendProjectMemory?.(project.id, kind, { text: entry, at: Date.now(), ownerId, ...(text(body.by) === '' ? {} : { by: text(body.by) }) })

core/src/host/store.js

@@ -518,7 +518,9 @@ export function install(rt) {
     // they were.
     const entryText = projectText(entry.text)
     if (entryText === '') throw projectRefusal('A Project memory entry needs text')
-    if (entryText.length > MAX_ENTRY) throw projectRefusal(`A Project memory entry is at most ${MAX_ENTRY} characters`)
+    // #1886: the bound counts code points, not UTF-16 units (`MAX_ENTRY` says why), so the emoji
+    // note the assistant writer accepts at 600 characters is accepted here too.
+    if ([...entryText].length > MAX_ENTRY) throw projectRefusal(`A Project memory entry is at most ${MAX_ENTRY} characters`)
     const table = memoryTable(project)
     const ownerId = projectText(entry.ownerId, 120) || primaryOwner(project)
     const record = ownerRow(table, ownerId, project)

core/test/project-assistant.test.js

@@ -111,6 +111,38 @@ test('memory is private to one Project and an unknown id is refused', async () =
   }
 })
 
+// #1886: the assistant writer and the store's `appendProjectMemory` must count the bound on the
+// same unit. Both now count code points, so the same emoji note is accepted by both and the same
+// 601-code-point one refused by both — the old `String#length` made a 350-emoji note look like 700.
+test('the assistant writer and the store count the memory bound in code points alike', async () => {
+  const { rt, stop } = await start()
+  try {
+    const project = await created(rt, { name: 'Points', ownerUserId: 'owner-1' })
+    const atCap = '😀'.repeat(600)
+    rt.projectAssistant.write(project.id, { kind: 'decision', text: atCap })
+    let memory = (await rt.projectEndpoint('project-memory', { id: project.id })).memory
+    assert.equal(memory.decisions.length, 1, 'the assistant writer accepts 600 code points')
+    assert.equal(memory.decisions[0].text, atCap, 'and the store keeps the entry whole')
+
+    const over = '😀'.repeat(601)
+    assert.throws(() => rt.projectAssistant.write(project.id, { kind: 'decision', text: over }),
+      error => error.code === 'bot/invalid' && error.message === 'A Project memory entry is at most 600 characters')
+    await assert.rejects(() => rt.projectEndpoint('project-memory-write', { id: project.id, kind: 'decision', text: over }),
+      error => error.code === 'bot/invalid' && error.message === 'A Project memory entry is at most 600 characters')
+    memory = (await rt.projectEndpoint('project-memory', { id: project.id })).memory
+    assert.equal(memory.decisions.length, 1, 'neither refusal adds a row')
+
+    const short = '😀'.repeat(350)
+    rt.projectAssistant.write(project.id, { kind: 'question', text: short })
+    await rt.projectEndpoint('project-memory-write', { id: project.id, kind: 'decision', text: short })
+    memory = (await rt.projectEndpoint('project-memory', { id: project.id })).memory
+    assert.deepEqual(memory.questions.map(entry => entry.text), [short], 'the assistant writer stores 350 emoji')
+    assert.deepEqual(memory.decisions.map(entry => entry.text), [atCap, short], 'and the store stores them too')
+  } finally {
+    await stop()
+  }
+})
+
 test('the assistant refuses an unknown Project and an archived one', async () => {
   const { rt, stop } = await start()
   try {

core/test/project-board-panels.test.js

@@ -376,6 +376,48 @@ test('#1847: the panel shows the Chinese refusal when the host refuses the memor
   }
 })
 
+// #1886: the bound was compared against `String#length`, which counts UTF-16 code units, so a 350
+// character emoji note — 700 units — was refused as "over 600 characters" even though the sentence
+// counts characters. The bound now counts code points, the unit the repo already uses for a
+// message's own length (`chat-index.js`, `busy-ack.js`).
+test('#1886: the memory bound counts code points, so an emoji note keeps its full 600 characters', async (t) => {
+  const rt = await startHost(t)
+  const call = (endpoint, payload) => rt.projectEndpoint(endpoint, payload)
+  const write = text => call('project-memory-write', { id: 'project-1', kind: 'decision', text })
+  const kept = async () => (await call('project-memory', { id: 'project-1' })).memory.decisions
+
+  assert.equal([...'😀'].length, 1, 'one emoji is one code point')
+  assert.equal('😀'.length, 2, 'and two UTF-16 units, which is what the bound used to count')
+
+  // The user-visible case: 350 characters is well short of 600, and 350 emoji are 700 units.
+  const short = '😀'.repeat(350)
+  await write(short)
+  let rows = await kept()
+  assert.equal(rows.length, 1, 'a 350-character emoji note is accepted')
+  assert.equal(rows[0].text, short, 'and stored whole, not cut')
+
+  // The cap is 600 code points: accepted, and 601 refused in the sentence the panel quotes.
+  const exact = '😀'.repeat(600)
+  await write(exact)
+  rows = await kept()
+  assert.deepEqual(rows.map(entry => entry.text), [short, exact], 'the entry at the cap is accepted')
+  await assert.rejects(() => write('😀'.repeat(601)),
+    error => error.code === 'bot/invalid' && error.message === 'A Project memory entry is at most 600 characters')
+  rows = await kept()
+  assert.equal(rows.length, 2, 'the refusal leaves no row behind')
+
+  // ASCII was already counted right; the new unit must leave it exactly as it was.
+  const ascii = 'x'.repeat(600)
+  await write(ascii)
+  rows = await kept()
+  assert.equal(rows.length, 3, 'an ASCII entry at the cap is still accepted')
+  assert.equal(rows[2].text, ascii, 'and still stored whole')
+  await assert.rejects(() => write('x'.repeat(601)),
+    error => error.code === 'bot/invalid' && error.message === 'A Project memory entry is at most 600 characters')
+  rows = await kept()
+  assert.equal(rows.length, 3, 'and ASCII over the cap still leaves nothing behind')
+})
+
 test("the settings panel shows the host's refusal instead of pretending the save landed", async () => {
   resetSlots()
   const refused = Object.assign(new Error('Only the owner of this Project may change it'), { code: 'bot/forbidden' })

docs/subsystems/core-host.md

@@ -50,7 +50,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core
 | `core/src/host/usage.js` | Usage ledger at `$DSH_HOME/bot/usage.sqlite` (`calls`). Ledger v3 (#903): every row carries `harness`, `issue_pr`, `tool_calls`, `wall_ms`, plus existing token columns (`withUsageFields` / `missingUsageFields`). Session turns are metered via `foldCalls` over turn events (`tool/call` count, wall time from attempt start, `issuePr` from turn text or `rt.noteUsageIssuePr(sessionId, ref)`), not from llm/stream alone. A triage tier is stamped with `rt.noteUsageHarness(sessionId, idOrLabels)` and lands on the row's `harness` field; the resume cursor (`marks.fold`) stays provider / model / retry. Sample row fixture: `core/test/fixtures/usage-sample.sqlite`. Tests: `core/test/usage-fields.test.js`, `core/test/usage.test.js`. |
 | `core/src/host/tool-schema.js` | The tool definitions the model sees: its `defineTool` wraps DSH's and completes the compiled schema with `explicitObjectShape`, which fills `properties: {}` and `required: []` on every object node (recursing through `properties`, `items`, `oneOf`/`anyOf`/`allOf`, `$defs`, …). A strict OpenAI-compatible gateway reads an absent member as null and rejects the whole request (`null is not of type "array"`), so one argument-less tool would break every turn; argument validation keeps using DSH's schema unchanged. Every host module registers tools through this wrapper. Tests: `core/test/tool-schema.test.js`. |
 | `core/src/host/project-state.js` | The `project_state` tool (#1702, #1550 Stage 2.4): the Project State files of the Session working folder (`PROJECT.md`, `ROADMAP.md`, `docs/governance.md`, `decisions/*.md`, `LESSONS.md`), listed with byte size, sha256 content hash and last-modified time, or one file handed over whole up to 200,000 bytes. A caller compares the hash with the copy it already had, so a stale copy is visible instead of assumed. Answers `absent`, `too large` (with the size) or `not a repository` rather than an empty document, refuses any path outside that list, and reads nothing at install time: no prompt, system message or Session context is written by a call. Tests: `core/test/project-state.test.js`. |
-| `core/src/host/project-assistant.js` | The Project Assistant (#1712, #1550 Stage 2.5): one hidden Bot and Session per Project, created by `project-create` through the roster and refusing an unknown or archived id, with exactly one 09:00 Asia/Shanghai task on that Session through the host's own `schedule-create` path and removed with the Project by `project-delete` — never a cron inside the plugin, and no backfill for a Project made before this slice. Memory lives in `state.projectMemory[id]` (`{decisions, questions, digest}`, behind `rt.projectMemoryOf` / `appendProjectMemory` / `setProjectDigest` and the `project-memory` / `project-memory-write` endpoints) keyed by Project id, so one Project cannot read another's entries. Since #1847 that bound is enforced twice, from one value: `MAX_ENTRY` (600, in `constants.js`) is checked by the assistant writer and again by `appendProjectMemory`, the one write door, so the endpoint refuses an empty entry (`A Project memory entry needs text`) and one over the cap (`A Project memory entry is at most 600 characters`) instead of storing a blank row or cutting the text, and a refusal leaves the table and `project.updatedAt` untouched; the panel quotes either sentence, in Chinese under a zh locale (`core/src/client/i18n.js`). `buildDigest` answers, in order, what merged since the previous digest, what closed, what is still in progress and on whom, and the next slice, using only board rows (`source`/`column`/`assignee`/`updatedAt`) and the Project State file list; a day with no change is one line, one day has one note and one `deliver()`, and the second build answers `skipped`. Identity is the install's GitHub connector account (the owner's own on a single-person install): with no connector signed in, or on a multi-member host whose `projects.ownerUserId` is not this host's own user, `digest` refuses and escalates instead of posting as the shared app, and every write carries `Written-by: Project Assistant (<name>)` and `Posted-for: <login>`. Since #1733 the memory and the digest are per (Project, owner) — `state.projectMemory[projectId].owners[ownerId]` with the primary owner mirrored on the row, behind `rt.projectMemoryFor(id, ownerId)`, `appendProjectMemory(id, kind, entry)`, `setProjectDigest(id, note, ownerId)` — and one `projectBoard` read plus one `listProjectState` read serve one render per owner, so the facts are shared and the personalisation is not; a read outside `ownersOf(project)` is refused naming that boundary. Since #1846 the digest reads the Project's own board, not one member's page: `rt.projectBoard(id, null, { internal: true })` is the one opt-in that skips the per-reader `taskVisible` rule, and the channel builds its options from the zone alone, so no request body can ask for it. Tests: `core/test/project-assistant.test.js`, `core/test/project-digest-authz.test.js`. |
+| `core/src/host/project-assistant.js` | The Project Assistant (#1712, #1550 Stage 2.5): one hidden Bot and Session per Project, created by `project-create` through the roster and refusing an unknown or archived id, with exactly one 09:00 Asia/Shanghai task on that Session through the host's own `schedule-create` path and removed with the Project by `project-delete` — never a cron inside the plugin, and no backfill for a Project made before this slice. Memory lives in `state.projectMemory[id]` (`{decisions, questions, digest}`, behind `rt.projectMemoryOf` / `appendProjectMemory` / `setProjectDigest` and the `project-memory` / `project-memory-write` endpoints) keyed by Project id, so one Project cannot read another's entries. Since #1847 that bound is enforced twice, from one value: `MAX_ENTRY` (600, in `constants.js`) is checked by the assistant writer and again by `appendProjectMemory`, the one write door, and since #1886 both count code points (`[...text].length`), not UTF-16 units, so a 600-character emoji note is stored rather than refused as 1,200 units; the endpoint refuses an empty entry (`A Project memory entry needs text`) and one over the cap (`A Project memory entry is at most 600 characters`) instead of storing a blank row or cutting the text, and a refusal leaves the table and `project.updatedAt` untouched; the panel quotes either sentence, in Chinese under a zh locale (`core/src/client/i18n.js`). `buildDigest` answers, in order, what merged since the previous digest, what closed, what is still in progress and on whom, and the next slice, using only board rows (`source`/`column`/`assignee`/`updatedAt`) and the Project State file list; a day with no change is one line, one day has one note and one `deliver()`, and the second build answers `skipped`. Identity is the install's GitHub connector account (the owner's own on a single-person install): with no connector signed in, or on a multi-member host whose `projects.ownerUserId` is not this host's own user, `digest` refuses and escalates instead of posting as the shared app, and every write carries `Written-by: Project Assistant (<name>)` and `Posted-for: <login>`. Since #1733 the memory and the digest are per (Project, owner) — `state.projectMemory[projectId].owners[ownerId]` with the primary owner mirrored on the row, behind `rt.projectMemoryFor(id, ownerId)`, `appendProjectMemory(id, kind, entry)`, `setProjectDigest(id, note, ownerId)` — and one `projectBoard` read plus one `listProjectState` read serve one render per owner, so the facts are shared and the personalisation is not; a read outside `ownersOf(project)` is refused naming that boundary. Since #1846 the digest reads the Project's own board, not one member's page: `rt.projectBoard(id, null, { internal: true })` is the one opt-in that skips the per-reader `taskVisible` rule, and the channel builds its options from the zone alone, so no request body can ask for it. Tests: `core/test/project-assistant.test.js`, `core/test/project-digest-authz.test.js`. |
 | `core/src/host/project-chief.js` | The Project Chief (#1859): every Project gets its own Chief, created with the Project by `project-create` — never the user's main Chief acting part-time. `rt.projectChief` has the same shape as `rt.projectAssistant` (`create(projectId)`, `chiefOf(projectId)`, `remove(projectId)`), the record carries `chief: { botId, sessionId, createdAt }` beside `assistant`, and the spawned Bot is named `Project Chief (<name>)`, bound with `bindProject` so it carries `projectId`, and given instructions that name that one Project and forbid answering for any other (another Project's name is never quoted). A host without the seam logs once and answers `undefined`, so `project-create` still creates the Project with `chief: { error }`; `project-delete` retires the Chief through `deleteBot` and clears its binding. Tests: `core/test/projects.test.js`. |
 | `core/src/host/project-team.js` | The Project's team dossier (#1864), read-only. `rt.projectTeam(projectId)` answers `{ rows }` in the board's order — the Project's Chief, its assistant, then `roles.maintainers` / `roles.members` — with each row `{ botId, name, role, persona, model, color, createdAt, memory: { head, topics }, skills, rooms }`. `persona` is the Bot's `instructions`; `memory.head` is its own `MEMORY.md` (read through `rt.memoryDigest` in `memory.js`, which also supplies the topic names and never a topic body); `skills` is the attached capabilities the library records as `skill` (a pack contributes its skill children); `rooms` is only the rooms this Project owns. Every string and list is capped (`DOSSIER_LIMITS`: persona and memory head 4000, 64 topics, 64 skills, 32 rooms), a deleted Bot is a `null` row instead of an error for the whole answer, and an unknown Project answers `{ rows: [] }`. The endpoint sits beside `project-read` in `authz.js`'s `TASK_ENDPOINTS`, per the owner decision that persona and memory are shared by the Project's members. Tests: `core/test/project-team.test.js`. |
 | `core/src/shared/artifact.js` | The typed facts a workflow node takes in and hands out (#1272): `parseArtifactSpec` reads `check:tests-ok` / `issue.ready` / `text:topic` into `{ kind, name }` and refuses a spelling the vocabulary does not have; `artifact(spec, revision, { value })` is one instance, and `REVISION_RULES` says what the revision of each kind is (a verdict its SHA, a check the SHA it ran on, a PR its number and `headRefOid`). `freshness(input, currentRevision)` answers present-and-current separately from present, and a fact nobody could read is `unreadable`, never a match. Tests: `core/test/workflow.test.js`. |