diff --git a/changelog.d/1899-experience-text-code-points.fixed.md b/changelog.d/1899-experience-text-code-points.fixed.md new file mode 100644 index 000000000..b1e758b0a --- /dev/null +++ b/changelog.d/1899-experience-text-code-points.fixed.md @@ -0,0 +1 @@ +An incident entry's `trigger`, `wrong` and `right` are bounded at 600 characters counted as code points, not UTF-16 units ([#1899](https://github.com/ThinkFlowLab/FlyBot/issues/1899)): 300 emoji no longer fill the 600 budget, a 600-character emoji entry is stored rather than refused as 1,200 units, and an entry already on disk that is over 600 UTF-16 units but within 600 characters loads again through the same check on the read path. diff --git a/core/src/host/experience.js b/core/src/host/experience.js index 08c7eefd5..abf8d8d31 100644 --- a/core/src/host/experience.js +++ b/core/src/host/experience.js @@ -10,6 +10,7 @@ import { existsSync, lstatSync, readdirSync, readFileSync } from 'node:fs' import { dirname, join, resolve, sep } from 'node:path' import { fileURLToPath } from 'node:url' +import { codePoints } from './text.js' export const DEFAULT_EXPERIENCE_ROOT = join(dirname(fileURLToPath(import.meta.url)), '../../dev/experience') @@ -97,7 +98,8 @@ function validateEntry(entry, expected) { if (entry.seen !== undefined && (!Number.isInteger(entry.seen) || entry.seen < 1)) throw new Error('seen') for (const key of ['trigger', 'wrong', 'right']) { const text = entry[key] - if (typeof text !== 'string' || text.trim() === '' || text.length > TEXT_MAX) throw new Error(key) + // #1899: a code point costs one whatever its UTF-16 width, so 600 emoji fill 600. + if (typeof text !== 'string' || text.trim() === '' || codePoints(text) > TEXT_MAX) throw new Error(key) } if (entry.refs !== undefined) { if (!Array.isArray(entry.refs) || entry.refs.length > 20 || entry.refs.some(ref => typeof ref !== 'string' || !REF.test(ref))) { diff --git a/core/test/experience.test.js b/core/test/experience.test.js index e9567f9fc..9c9892bd7 100644 --- a/core/test/experience.test.js +++ b/core/test/experience.test.js @@ -256,3 +256,16 @@ test('the CLI writes one new entry file and refuses to overwrite it', () => { rmSync(root, { recursive: true, force: true }) } }) + +test('the experience text bound counts code points, not UTF-16 units', () => { + const emoji = '\u{1F600}' + const base = { id: 'bound', repo: 'Alpha/One', techStacks: [], date: '2026-10-31', trigger: 'a', wrong: 'b', right: 'c' } + for (const key of ['trigger', 'wrong', 'right']) { + for (const text of ['x'.repeat(599), 'x'.repeat(600), emoji.repeat(599), emoji.repeat(600), `${emoji.repeat(599)}x`]) { + assert.doesNotThrow(() => prepareExperienceEntry({ ...base, [key]: text }), `${key}: ${text.length} units is within 600 characters`) + } + for (const text of ['x'.repeat(601), emoji.repeat(601), `${emoji.repeat(600)}x`]) { + assert.throws(() => prepareExperienceEntry({ ...base, [key]: text }), new RegExp(`^Error: ${key}$`), `${key}: over 600 characters is refused`) + } + } +}) diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 339daea52..f45788ce5 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -47,7 +47,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | `core/src/host/automation-default.js` | Automation tasks on by default (#378): once per profile, `offerBundle` asks DSH's plugin manager to switch on `@deepseek-ai/dsh-experimental-schedule-bundle` and writes the offer to `dsh-defaults.json` next to the team. Tests: `core/test/automation-default.test.js`. | | `core/src/host/notify.js` | Out-of-app delivery, first half (#355): a transport-agnostic adapter with Slack's `chat.postMessage` shape, one key per reminder so a retried send cannot duplicate, exponential backoff that honours `Retry-After`, three attempts at most, a structural failure not retried, and zero transport calls while the channel is off. Ships `fakeSlack` for tests; nothing calls it until the real Slack transport lands (see **Out-of-app delivery, first half**). Tests: `core/test/notify.test.js`. | | `core/src/host/harness.js` | Declarative harness (#874 Option B). Step 0 (#903): `default` names today's prompt / tools / model / step·retry / verifiers (`rt.harness`, `rt.resolveHarness`); behavior-neutral. Step 1: `lite` / `medium` / `heavy` load through the same resolver with distinct declared budgets; `rt.routeHarness` reads one triage label (`harness:lite`, `harness:medium`, `harness:heavy`). `applied` stays false, so the named modules are not rewritten. `harnessPro` checks a config offline against CI, `core/dev/eval`, and `rsi-baseline` and does not call a model. Step 2: `rt.harnessExperience` reads the experience library and does not attach it to a tier. Tree prompt exception: `@flybot/tree` keeps its marked prompt under `plugins/tree` (outside the three mirrors); listed on the harness, injection stays outside this file. Installed in `MODULES` right after `store`. Tests: `core/test/harness.test.js`, `core/test/experience.test.js`. | -| `core/src/host/experience.js` | Experience library read API (#874 Step 2). `lookupExperience({ repo, techStack, query, root })` reads `core/dev/experience/`. Data and `check.mjs` stay under `core/dev/` (see [ds-bot-core](ds-bot-core.md)). | +| `core/src/host/experience.js` | Experience library read API (#874 Step 2). `lookupExperience({ repo, techStack, query, root })` reads `core/dev/experience/`. Data and `check.mjs` stay under `core/dev/` (see [ds-bot-core](ds-bot-core.md)). Since #1899 the 600-character bound on `trigger`, `wrong` and `right` counts code points through `codePoints` (`text.js`), so an emoji costs one character, not two, and an entry written before this change whose text is over 600 UTF-16 units but within 600 characters still loads through the read path. | | `core/src/host/usage.js` | Usage ledger at `$DSH_HOME/bot/usage.sqlite` (`calls`). Ledger v3 (#903): every row carries `harness`, `issue_pr`, `tool_calls`, `wall_ms`, plus existing token columns (`withUsageFields` / `missingUsageFields`). Session turns are metered via `foldCalls` over turn events (`tool/call` count, wall time from attempt start, `issuePr` from turn text or `rt.noteUsageIssuePr(sessionId, ref)`), not from llm/stream alone. A triage tier is stamped with `rt.noteUsageHarness(sessionId, idOrLabels)` and lands on the row's `harness` field; the resume cursor (`marks.fold`) stays provider / model / retry. Sample row fixture: `core/test/fixtures/usage-sample.sqlite`. Tests: `core/test/usage-fields.test.js`, `core/test/usage.test.js`. | | `core/src/host/tool-schema.js` | The tool definitions the model sees: its `defineTool` wraps DSH's and completes the compiled schema with `explicitObjectShape`, which fills `properties: {}` and `required: []` on every object node (recursing through `properties`, `items`, `oneOf`/`anyOf`/`allOf`, `$defs`, …). A strict OpenAI-compatible gateway reads an absent member as null and rejects the whole request (`null is not of type "array"`), so one argument-less tool would break every turn; argument validation keeps using DSH's schema unchanged. Every host module registers tools through this wrapper. Tests: `core/test/tool-schema.test.js`. | | `core/src/host/project-state.js` | The `project_state` tool (#1702, #1550 Stage 2.4): the Project State files of the Session working folder (`PROJECT.md`, `ROADMAP.md`, `docs/governance.md`, `decisions/*.md`, `LESSONS.md`), listed with byte size, sha256 content hash and last-modified time, or one file handed over whole up to 200,000 bytes. A caller compares the hash with the copy it already had, so a stale copy is visible instead of assumed. Answers `absent`, `too large` (with the size) or `not a repository` rather than an empty document, refuses any path outside that list, and reads nothing at install time: no prompt, system message or Session context is written by a call. Tests: `core/test/project-state.test.js`. | diff --git a/patches/ds-bot/files/src/host/experience.js b/patches/ds-bot/files/src/host/experience.js index 08c7eefd5..abf8d8d31 100644 --- a/patches/ds-bot/files/src/host/experience.js +++ b/patches/ds-bot/files/src/host/experience.js @@ -10,6 +10,7 @@ import { existsSync, lstatSync, readdirSync, readFileSync } from 'node:fs' import { dirname, join, resolve, sep } from 'node:path' import { fileURLToPath } from 'node:url' +import { codePoints } from './text.js' export const DEFAULT_EXPERIENCE_ROOT = join(dirname(fileURLToPath(import.meta.url)), '../../dev/experience') @@ -97,7 +98,8 @@ function validateEntry(entry, expected) { if (entry.seen !== undefined && (!Number.isInteger(entry.seen) || entry.seen < 1)) throw new Error('seen') for (const key of ['trigger', 'wrong', 'right']) { const text = entry[key] - if (typeof text !== 'string' || text.trim() === '' || text.length > TEXT_MAX) throw new Error(key) + // #1899: a code point costs one whatever its UTF-16 width, so 600 emoji fill 600. + if (typeof text !== 'string' || text.trim() === '' || codePoints(text) > TEXT_MAX) throw new Error(key) } if (entry.refs !== undefined) { if (!Array.isArray(entry.refs) || entry.refs.length > 20 || entry.refs.some(ref => typeof ref !== 'string' || !REF.test(ref))) { diff --git a/patches/ds-bot/files/test/experience.test.js b/patches/ds-bot/files/test/experience.test.js index e9567f9fc..9c9892bd7 100644 --- a/patches/ds-bot/files/test/experience.test.js +++ b/patches/ds-bot/files/test/experience.test.js @@ -256,3 +256,16 @@ test('the CLI writes one new entry file and refuses to overwrite it', () => { rmSync(root, { recursive: true, force: true }) } }) + +test('the experience text bound counts code points, not UTF-16 units', () => { + const emoji = '\u{1F600}' + const base = { id: 'bound', repo: 'Alpha/One', techStacks: [], date: '2026-10-31', trigger: 'a', wrong: 'b', right: 'c' } + for (const key of ['trigger', 'wrong', 'right']) { + for (const text of ['x'.repeat(599), 'x'.repeat(600), emoji.repeat(599), emoji.repeat(600), `${emoji.repeat(599)}x`]) { + assert.doesNotThrow(() => prepareExperienceEntry({ ...base, [key]: text }), `${key}: ${text.length} units is within 600 characters`) + } + for (const text of ['x'.repeat(601), emoji.repeat(601), `${emoji.repeat(600)}x`]) { + assert.throws(() => prepareExperienceEntry({ ...base, [key]: text }), new RegExp(`^Error: ${key}$`), `${key}: over 600 characters is refused`) + } + } +})