diff --git a/changelog.d/1895-host-limits-code-points.fixed.md b/changelog.d/1895-host-limits-code-points.fixed.md new file mode 100644 index 000000000..260fbcbed --- /dev/null +++ b/changelog.d/1895-host-limits-code-points.fixed.md @@ -0,0 +1 @@ +Host character limits count code points, so a character outside the BMP costs one instead of two ([#1895](https://github.com/ThinkFlowLab/FlyBot/issues/1895)): a memory entry at 300, a `MEMORY.md` at 4,000 and its size note, a scheduled task's name at 120, a Bot-to-Bot message at 20,000, and the Project memory entry at 600 in both writers now measure what a reader counts, and a refusal prints that code-point number. diff --git a/core/src/host/hooks.js b/core/src/host/hooks.js index 2d8b9e021..96f933d5b 100644 --- a/core/src/host/hooks.js +++ b/core/src/host/hooks.js @@ -23,6 +23,7 @@ import { FEED_EVENTS, installFeed } from './flybot-feed.js' import { createPluginModel } from './plugin-model.js' import { defineTool } from './tool-schema.js' import { retiredBot } from './store.js' +import { codePoints } from './text.js' import { EPHEMERAL_DENIED_TOOLS } from './tools.js' export const FLYBOT_API_VERSION = 1 @@ -728,7 +729,8 @@ export function install(rt) { async function send({ from, to, text, role = 'request', hop = 0, via = SHARED, meta, attachments } = {}) { await rt.load() if (typeof text !== 'string' || text.trim() === '') throw new Error('flybot: send needs text') - if (text.length > MAX_SEND_TEXT) throw new Error(`flybot: a message is at most ${MAX_SEND_TEXT} characters`) + // #1895: the bound counts code points, so a message of emoji is measured as a reader counts it. + if (codePoints(text) > MAX_SEND_TEXT) throw new Error(`flybot: a message is at most ${MAX_SEND_TEXT} characters`) if (role !== 'request' && role !== 'reply') throw new Error(`flybot: unsupported role ${role}`) if (!isRemote(from) && rt.botOf(from) === undefined) throw new Error(`flybot: unknown sender ${from}`) if (!isRemote(to) && rt.botOf(to) === undefined) throw new Error(`flybot: unknown Bot ${to}`) diff --git a/core/src/host/memory.js b/core/src/host/memory.js index 57123eb4c..17bbc968e 100644 --- a/core/src/host/memory.js +++ b/core/src/host/memory.js @@ -27,7 +27,7 @@ import { dirname, join } from 'node:path' import { BlockAssembler } from '@deepseek-ai/dsh-llm' import { currentParticipants } from './room-membership.js' import { defineTool } from './tool-schema.js' -import { splitRef, unquote } from './text.js' +import { codePoints, splitRef, unquote } from './text.js' import { output } from './tools.js' import { retiredBot } from './store.js' import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js' @@ -530,7 +530,10 @@ export function install(rt) { const secretEntry = text => (rt.redactSecrets?.(text) ?? text) !== text || looksSecret(text) function entryProblem(text) { if (text === '') return 'Write the entry to save.' - if (text.length > MEMORY_LIMITS.entry) return `An entry holds at most ${MEMORY_LIMITS.entry} characters; this one has ${text.length}. Keep one fact per entry, or save the details as several entries in a topic.` + // #1895: one character costs one whatever its UTF-16 width, and the number the refusal + // prints is the one a reader would count. + const size = codePoints(text) + if (size > MEMORY_LIMITS.entry) return `An entry holds at most ${MEMORY_LIMITS.entry} characters; this one has ${size}. Keep one fact per entry, or save the details as several entries in a topic.` if (secretEntry(text)) return 'This looks like a key, token, password, or ID number, which memory never keeps. Keys go to request_secret.' return undefined } @@ -540,14 +543,17 @@ export function install(rt) { function checkSize(file, next, label) { const limit = limitOf(file) + const size = codePoints(next) // A file already past its size (edited by hand) may still shrink. - if (next.length <= limit || next.length <= file.content.length) return + if (size <= limit || size <= codePoints(file.content)) return throw new Error(file.topic === undefined - ? `${capital(label)}'s MEMORY.md would have ${next.length} of its ${limit} characters. Make room first: move details into a topic (remember with topic, then forget them here), merge entries with replace, or forget what no longer matters.` - : `The topic [[${file.topic}]] in ${label} would have ${next.length} of its ${limit} characters. Put the rest in another topic, or forget old entries.`) + ? `${capital(label)}'s MEMORY.md would have ${size} of its ${limit} characters. Make room first: move details into a topic (remember with topic, then forget them here), merge entries with replace, or forget what no longer matters.` + : `The topic [[${file.topic}]] in ${label} would have ${size} of its ${limit} characters. Put the rest in another topic, or forget old entries.`) } const capital = text => text[0].toUpperCase() + text.slice(1) - const sizeNote = (file, size) => `${file.file}: ${size} of ${limitOf(file)} characters` + // #1895: both the bound and the number it prints count code points, so a file of emoji is + // measured the way a reader sees it. + const sizeNote = (file, text) => `${file.file}: ${codePoints(text)} of ${limitOf(file)} characters` const allEntries = files => files.flatMap(file => file.entries.map(entry => ({ ...entry, file }))) const actorOf = actor => ({ actor: actor.id, actorName: actor.name, ...(actor.auto ? { auto: true } : {}) }) @@ -589,7 +595,7 @@ export function install(rt) { await journal({ op: 'add', scope: scopeKey(scope), file: target.file, ...actorOf(actor), after: line }) noteNews(scope, actor, 'add', line) const where = topic === undefined ? MAIN : fresh ? `the new topic [[${topic}]], linked from MEMORY.md` : `topic [[${topic}]]` - return { changed: true, file: target, message: `Saved to ${label}, in ${where} (${sizeNote(target, next.length)}).` } + return { changed: true, file: target, message: `Saved to ${label}, in ${where} (${sizeNote(target, next)}).` } }) } @@ -879,7 +885,7 @@ export function install(rt) { const topic = cleanTopic(args.topic) for (const { label, files } of loaded) { const file = files.find(item => item.topic === topic) - if (file !== undefined) return `Topic [[${topic}]] in ${label} (${sizeNote(file, file.content.length)}):\n${file.content.trim() || '(empty)'}` + if (file !== undefined) return `Topic [[${topic}]] in ${label} (${sizeNote(file, file.content)}):\n${file.content.trim() || '(empty)'}` } const known = loaded.flatMap(({ tag, files }) => files.filter(file => file.topic).map(file => `[[${file.topic}]] (${tag})`)) return `No topic [[${String(args.topic).slice(0, 60)}]] there. Topics: ${known.join(', ') || 'none yet'}.` @@ -904,7 +910,7 @@ export function install(rt) { return loaded.map(({ tag, label, files }) => { const [main, ...topics] = files return [ - `${capital(label)}: ${MAIN} has ${main.entries.length} entries (${sizeNote(main, main.content.length)})${topics.length > 0 ? `; topics ${topics.map(file => `[[${file.topic}]] (${file.entries.length})`).join(', ')}` : '; no topics'}.`, + `${capital(label)}: ${MAIN} has ${main.entries.length} entries (${sizeNote(main, main.content)})${topics.length > 0 ? `; topics ${topics.map(file => `[[${file.topic}]] (${file.entries.length})`).join(', ')}` : '; no topics'}.`, ...main.entries.map(entry => `[${where(tag, main)}] ${formatEntry(entry.text, entry.meta).slice(2)}`), ].join('\n') }).join('\n\n') @@ -1062,7 +1068,7 @@ export function install(rt) { return { botId: bot.id, entries, - sizes: { own: own[0].content.length, team: team[0].content.length }, + sizes: { own: codePoints(own[0].content), team: codePoints(team[0].content) }, limits: MEMORY_LIMITS, summary: entries.length > 0 && saved !== undefined ? { text: saved.text, at: saved.at } : null, fresh: entries.length === 0 || saved?.hash === hash, @@ -1324,7 +1330,7 @@ export function install(rt) { const teamFiles = loadScope(TEAM) const said = read.filter(line => line.who === 'user').map(line => line.text).join('\n') const { numbered, left } = reviewEntries(ownFiles, teamFiles, REVIEW_MEMORY_MAX, said) - const sizes = { own: ownFiles[0].content.length, team: teamFiles[0].content.length } + const sizes = { own: codePoints(ownFiles[0].content), team: codePoints(teamFiles[0].content) } const earlier = batch.earlier.map(line => reviewLine(line.who, line.text, bot)) const language = writtenIn(read.filter(line => line.who === 'user').map(line => line.text.replace(/^User: /, '')), languageNow()) const reply = await botShot(bot, reviewSystem(writesTeam(bot.id)), reviewRequest({ bot, numbered, left, sizes, where: batch.where, earlier, fresh: fresh.texts, more, language }), { sessionId: rt.chatOf(bot.id), kind: 'memory' }) diff --git a/core/src/host/project-assistant.js b/core/src/host/project-assistant.js index ca3995a01..6a37f5867 100644 --- a/core/src/host/project-assistant.js +++ b/core/src/host/project-assistant.js @@ -17,6 +17,7 @@ import { defineTool } from './tool-schema.js' import { MAX_ENTRY } from './constants.js' import { listProjectState } from './project-state.js' +import { codePoints } from './text.js' import { output } from './tools.js' import { inQuietHours, clockIn, quietHoursOf, readQuietHours } from '../shared/quiet-hours.js' @@ -190,7 +191,8 @@ export function install(rt) { if (entry === '') throw refusal('A Project memory entry needs text') // #1886: count code points, the same unit `appendProjectMemory` counts, so the two writers // cannot disagree about which note fits (`String#length` charged an emoji two units). - if ([...entry].length > MAX_ENTRY) throw refusal(`A Project memory entry is at most ${MAX_ENTRY} characters`) + // #1895: both count through the one `codePoints` helper. + if (codePoints(entry) > MAX_ENTRY) throw refusal(`A Project memory entry is at most ${MAX_ENTRY} characters`) const ownerId = text(body.ownerId) || ownersOf(project)[0] || '' if (text(body.ownerId) !== '' && !ownersOf(project).includes(text(body.ownerId))) throw outsideBoundary(project, body.ownerId) rt.appendProjectMemory?.(project.id, kind, { text: entry, at: Date.now(), ownerId, ...(text(body.by) === '' ? {} : { by: text(body.by) }) }) diff --git a/core/src/host/schedules.js b/core/src/host/schedules.js index 10a9408e9..8092e3700 100644 --- a/core/src/host/schedules.js +++ b/core/src/host/schedules.js @@ -31,7 +31,7 @@ import { randomUUID } from 'node:crypto' import { createUserMessage } from '@deepseek-ai/dsh-llm' import { quietHoursOf, inQuietHours, clockIn, quietError } from '../shared/quiet-hours.js' import { defineTool } from './tool-schema.js' -import { textOf } from './text.js' +import { codePoints, textOf } from './text.js' import { output } from './tools.js' import { retiredBot } from './store.js' @@ -335,7 +335,8 @@ export function install(rt) { const title = String(body.title ?? '').trim() const prompt = String(body.prompt ?? '').trim() if (title === '') throw invalid('Give the task a name') - if (title.length > MAX_TITLE) throw invalid(`Keep the name under ${MAX_TITLE} characters`) + // #1895: the bound counts code points, so a name of emoji is measured as a reader counts it. + if (codePoints(title) > MAX_TITLE) throw invalid(`Keep the name under ${MAX_TITLE} characters`) if (prompt === '') throw invalid('Say what the Bot should do') return { sessionId: bot ? rt.chatOf(bot.id) : room.id, title, prompt } } diff --git a/core/src/host/store.js b/core/src/host/store.js index c2bf8cbbf..985759b16 100644 --- a/core/src/host/store.js +++ b/core/src/host/store.js @@ -8,7 +8,7 @@ import { join } from 'node:path' import { LEGACY_THEMES, MAX_ENTRY, MAX_EXCHANGES, ROOM_MODES } from './constants.js' import { homeLock } from './home-lock.js' import { openJournal } from './journal.js' -import { currentColor, normalize, unquote } from './text.js' +import { codePoints, currentColor, normalize, unquote } from './text.js' // The saved team's format version. Version 2 (#47) adds the identity schema: member // records, a per-Bot owner binding, and the owner's designated Chief. A version-1 file @@ -519,8 +519,9 @@ export function install(rt) { const entryText = projectText(entry.text) if (entryText === '') throw projectRefusal('A Project memory entry needs text') // #1886: the bound counts code points, not UTF-16 units (`MAX_ENTRY` says why), so the emoji - // note the assistant writer accepts at 600 characters is accepted here too. - if ([...entryText].length > MAX_ENTRY) throw projectRefusal(`A Project memory entry is at most ${MAX_ENTRY} characters`) + // note the assistant writer accepts at 600 characters is accepted here too. #1895: both + // writers count through the one `codePoints` helper. + if (codePoints(entryText) > MAX_ENTRY) throw projectRefusal(`A Project memory entry is at most ${MAX_ENTRY} characters`) const table = memoryTable(project) const ownerId = projectText(entry.ownerId, 120) || primaryOwner(project) const record = ownerRow(table, ownerId, project) diff --git a/core/src/host/text.js b/core/src/host/text.js index db3dcb8c9..28f4ab8bc 100644 --- a/core/src/host/text.js +++ b/core/src/host/text.js @@ -23,6 +23,17 @@ export const unquote = text => String(text ?? '').trim().replace(/^["'β€œβ€γ€Œ export const escapeRegExp = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +// The host bounds this change counts go through it (#1895): one character costs one +// whatever its UTF-16 width, so an emoji inside a 300-character bound is one character, +// not two. +export const codePoints = (text) => { + const value = String(text) + let n = 0 + // A code point is one or two UTF-16 units; step by that width so no array is built. + for (let i = 0; i < value.length; n += 1) i += value.codePointAt(i) > 0xffff ? 2 : 1 + return n +} + // The part of a long message around the first match, so a search hit stays visible. export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => { if (text.length <= max) return text diff --git a/core/test/flybot.test.js b/core/test/flybot.test.js index 557c555db..508581262 100644 --- a/core/test/flybot.test.js +++ b/core/test/flybot.test.js @@ -1037,3 +1037,13 @@ test('#587: list_bots marks a Bot whose send waits on the owner with a count, ne const after = await tool('list_bots', {}, 'a') assert.ok(!after.includes('to approve'), 'a decided hold leaves no tag') }) + +// #1895: a message's 20,000-character bound counts code points, so an emoji message is +// measured the way a reader sees it instead of at double length. +test('a message counts code points against its 20,000-character bound (#1895)', async (t) => { + const { fb, stop } = await start() + t.after(stop) + const atCap = 'πŸ˜€'.repeat(20_000) + assert.equal((await fb.bot.send({ from: 'a', to: 'b', text: atCap })).action, 'allow', '20,000 emoji is 20,000 characters') + await assert.rejects(fb.bot.send({ from: 'a', to: 'b', text: 'πŸ˜€'.repeat(20_001) }), /a message is at most 20000 characters/, 'one code point over is refused') +}) diff --git a/core/test/memory.test.js b/core/test/memory.test.js index 3453db1be..b32718226 100644 --- a/core/test/memory.test.js +++ b/core/test/memory.test.js @@ -1176,3 +1176,38 @@ test('recall related lists what a topic links to', async (t) => { assert.match(out, /- \[\[memory\]\] β€” linked/) assert.match(await tool('recall', { related: 'nothinghere' }, 'b2'), /Nothing in your memory and the team memory links "nothinghere"/) }) + +// #1895: a character outside the BMP is one character, not two. The entry bound counts +// code points, and a refusal prints the number a reader would count. +test('the entry bound counts code points (#1895)', async (t) => { + const { tool, stop } = start() + t.after(() => stop()) + const atCap = 'πŸ˜€'.repeat(MEMORY_LIMITS.entry) + assert.match(await tool('remember', { text: atCap }, 'b2'), /^Saved to your memory/, '300 emoji is 300 characters') + const over = 'πŸ˜€'.repeat(MEMORY_LIMITS.entry + 1) + assert.match(await tool('remember', { text: over }, 'b2'), /at most 300 characters; this one has 301\./, 'the refusal counts code points') +}) + +// #1895: a MEMORY.md at its 4,000-character size is measured in code points too, and the +// size note prints what a reader would count β€” not twice the emoji count. +test('a MEMORY.md size and its note count code points (#1895)', async (t) => { + const { tool, read, stop, home } = start() + t.after(() => stop()) + const dir = join(home, 'memory', 'bots', 'b2') + const file = join(dir, 'MEMORY.md') + const withEmoji = count => `# Memory\n\n- ${'πŸ˜€'.repeat(count)}\n` + mkdirSync(dir, { recursive: true }) + + // A hand-written file of 3,943 code points (about 7,900 UTF-16 units) takes one more entry. + writeFileSync(file, withEmoji(3930)) + const savedMessage = await tool('remember', { text: 'One more' }, 'b2') + assert.match(savedMessage, /^Saved to your memory/, 'a file under 4,000 code points takes an entry') + const saved = [...read('bots', 'b2', 'MEMORY.md')].length + const added = saved - [...withEmoji(3930)].length + assert.ok(saved <= 4000, String(saved)) + assert.match(savedMessage, new RegExp(`MEMORY\\.md: ${saved} of 4000 characters`), 'the note prints the code-point count') + // Then one code point more in the file puts the next entry one over the bound, and the + // refusal prints that code-point number rather than twice it. + writeFileSync(file, withEmoji(4001 - added - 13)) + assert.match(await tool('remember', { text: 'One more' }, 'b2'), /MEMORY\.md would have 4001 of its 4000 characters/, 'one code point over is refused with the code-point count') +}) diff --git a/core/test/schedules.test.js b/core/test/schedules.test.js index 0b42befc6..97baea6ee 100644 --- a/core/test/schedules.test.js +++ b/core/test/schedules.test.js @@ -921,3 +921,19 @@ test('#1321: with nobody signed in the single-human path still copies across Bot assert.equal(copied.ok, true, JSON.stringify(copied)) assert.equal([...schedule.tasks.values()].at(-1).sessionId, 'b1') }) + +// #1895: a task name's 120-character bound counts code points, so a name of emoji is +// measured the way a reader sees it instead of at double length. +test('a task name counts code points against its 120-character bound (#1895)', async (t) => { + const schedule = scheduleService() + const { ctx, home, call } = setup({ schedule }) + t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) }) + const atCap = 'πŸ˜€'.repeat(120) + const made = await call('schedule-create', { target: 'm1', title: atCap, prompt: 'Go', when: { kind: 'every', minutes: 5 } }) + assert.equal(made.ok, true, JSON.stringify(made)) + assert.equal([...schedule.tasks.values()].at(-1).record.title, atCap, 'the whole 120-emoji name reaches DSH') + const over = 'πŸ˜€'.repeat(121) + const refused = await call('schedule-create', { target: 'm1', title: over, prompt: 'Go', when: { kind: 'every', minutes: 5 } }) + assert.equal(refused.ok, false) + assert.match(refused.error.message, /Keep the name under 120 characters/) +}) diff --git a/core/test/text.test.js b/core/test/text.test.js index 32462296f..1f99d8181 100644 --- a/core/test/text.test.js +++ b/core/test/text.test.js @@ -1,6 +1,18 @@ import assert from 'node:assert/strict' import { test } from 'node:test' -import { clip, createClock, escapeRegExp, estimateTokens, excerpt, isColor, normalize, splitRef, textOf, unquote } from '../src/host/text.js' +import { clip, codePoints, createClock, escapeRegExp, estimateTokens, excerpt, isColor, normalize, splitRef, textOf, unquote } from '../src/host/text.js' + +// #1895: the one helper every host limit counts with, so a character outside the BMP is +// one character and never two. +test('codePoints counts one character as one whatever its UTF-16 width (#1895)', () => { + assert.equal(codePoints(''), 0) + assert.equal(codePoints('abc'), 3) + assert.equal(codePoints('πŸ˜€'), 1, 'an emoji is one, though its UTF-16 length is 2') + assert.equal('πŸ˜€'.length, 2, 'the unit the limits used to count it in') + assert.equal(codePoints('aπŸ˜€b'), 3) + assert.equal(codePoints('ζ—…θ‘Œ'), 2) + assert.equal(codePoints(`a${'πŸ˜€'.repeat(4)}`), 5) +}) test('textOf joins the text blocks only', () => { assert.equal(textOf([{ type: 'text', text: ' a ' }, { type: 'image' }, { type: 'text', text: 'b' }, { type: 'text', text: 3 }]), 'a \nb') diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 0dfcb4716..339daea52 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -38,11 +38,12 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | --- | --- | | `core/index.js` | Server entry. | | `core/src/host/` | Host-side helpers (non-hooks). | +| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (`[...text].length`) is how the host counts a limit β€” one character costs one whatever its UTF-16 width β€” and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. | | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative β€” an addressed Bot answers as before. | | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. | | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `'s Chief` / ` Β· 's Bot`, ` Β· `, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). | | `core/src/host/reach.js` | Per-Bot grants and the owner-policy `reach()` check that hooks.js runs outside the plugin chain (#90); `rt.setGrants`, `rt.hasGrant`, `rt.hasAnyGrant`. A local Bot refused toward a friend's Bot is told that it is not shared with that friend and where to share it (Sharing, #273). `rt.reach` also reads the room a send already belongs to when the caller names none (#948): `sharedRoom` applies only to the `roomOnly` identity β€” the Bot holds no grant at all and came into that **face-to-face** room by its number (`isCodeJoined`) β€” and looks for the one such room where the **exact** target is still seated; a Bot that holds a grant, or a room of this home, is judged as it always was and keeps asking for an explicit `roomId`, so a Bot that joined a room by number β€” a `roomOnly` identity with no person-grant β€” may line that seat, which is the same allowance `reach()` already applies to an explicit `roomId`; the room itself is named outbound by the transport, never assumed here. A `user-mention` with no named room is also judged under `mentionRoom` (#1126): one live local or inbound group where that Bot is a member and the exact target is still a remote seat, including a seat a group invite admitted (`admittedBy: 'ticket'`). A plain line, `message_bot`, a seat that has left, and a stranger outside every such room stay on the grant rule. Face-to-face stays on `sharedRoom`. Tests: `core/test/reach-policy.test.js`, `core/test/flybot.test.js`. | -| `core/src/host/schedules.js` | Scheduled tasks through DSH's schedule service: list, create, edit, copy, pause/resume, retry and delete for Settings β†’ Scheduled tasks, each task's last run, and moving a Bot's tasks to its fresh part, which rewrites the stored row in place so the task keeps its id, its delivery history and the instant it was already due (#354, #473, #1245, #1287, see **Scheduled tasks: edit, pause, missed runs**, **Scheduled tasks: copy** and **Scheduled tasks: last run and Retry**). Also the `bot_schedules` tool: the Bot-wide list DSH's own `schedule_list` cannot give, group chats' own Sessions included (#628, see **Scheduled tasks: the Bot-wide list**). Tests: `core/test/schedules.test.js`. Tests for team quiet hours, its refusals and the settings write: `core/test/quiet-hours.test.js`; the rule itself is in `core/src/shared/quiet-hours.js` (#1166). The `every` selector takes `seconds` (60 to 604800) as well as the older `minutes` form, so a 90-second repeat survives an edit; a value below 60s or above 7 days is refused with the same sentence (#1651). | +| `core/src/host/schedules.js` | Scheduled tasks through DSH's schedule service: list, create, edit, copy, pause/resume, retry and delete for Settings β†’ Scheduled tasks, each task's last run, and moving a Bot's tasks to its fresh part, which rewrites the stored row in place so the task keeps its id, its delivery history and the instant it was already due (#354, #473, #1245, #1287, see **Scheduled tasks: edit, pause, missed runs**, **Scheduled tasks: copy** and **Scheduled tasks: last run and Retry**). Also the `bot_schedules` tool: the Bot-wide list DSH's own `schedule_list` cannot give, group chats' own Sessions included (#628, see **Scheduled tasks: the Bot-wide list**). Tests: `core/test/schedules.test.js`. Tests for team quiet hours, its refusals and the settings write: `core/test/quiet-hours.test.js`; the rule itself is in `core/src/shared/quiet-hours.js` (#1166). The `every` selector takes `seconds` (60 to 604800) as well as the older `minutes` form, so a 90-second repeat survives an edit; a value below 60s or above 7 days is refused with the same sentence (#1651). A task name is at most 120 code points (#1895), so 120 emoji is a valid name and one more is refused. | | `core/src/host/automation-default.js` | Automation tasks on by default (#378): once per profile, `offerBundle` asks DSH's plugin manager to switch on `@deepseek-ai/dsh-experimental-schedule-bundle` and writes the offer to `dsh-defaults.json` next to the team. Tests: `core/test/automation-default.test.js`. | | `core/src/host/notify.js` | Out-of-app delivery, first half (#355): a transport-agnostic adapter with Slack's `chat.postMessage` shape, one key per reminder so a retried send cannot duplicate, exponential backoff that honours `Retry-After`, three attempts at most, a structural failure not retried, and zero transport calls while the channel is off. Ships `fakeSlack` for tests; nothing calls it until the real Slack transport lands (see **Out-of-app delivery, first half**). Tests: `core/test/notify.test.js`. | | `core/src/host/harness.js` | Declarative harness (#874 Option B). Step 0 (#903): `default` names today's prompt / tools / model / stepΒ·retry / verifiers (`rt.harness`, `rt.resolveHarness`); behavior-neutral. Step 1: `lite` / `medium` / `heavy` load through the same resolver with distinct declared budgets; `rt.routeHarness` reads one triage label (`harness:lite`, `harness:medium`, `harness:heavy`). `applied` stays false, so the named modules are not rewritten. `harnessPro` checks a config offline against CI, `core/dev/eval`, and `rsi-baseline` and does not call a model. Step 2: `rt.harnessExperience` reads the experience library and does not attach it to a tier. Tree prompt exception: `@flybot/tree` keeps its marked prompt under `plugins/tree` (outside the three mirrors); listed on the harness, injection stays outside this file. Installed in `MODULES` right after `store`. Tests: `core/test/harness.test.js`, `core/test/experience.test.js`. | @@ -50,7 +51,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | `core/src/host/usage.js` | Usage ledger at `$DSH_HOME/bot/usage.sqlite` (`calls`). Ledger v3 (#903): every row carries `harness`, `issue_pr`, `tool_calls`, `wall_ms`, plus existing token columns (`withUsageFields` / `missingUsageFields`). Session turns are metered via `foldCalls` over turn events (`tool/call` count, wall time from attempt start, `issuePr` from turn text or `rt.noteUsageIssuePr(sessionId, ref)`), not from llm/stream alone. A triage tier is stamped with `rt.noteUsageHarness(sessionId, idOrLabels)` and lands on the row's `harness` field; the resume cursor (`marks.fold`) stays provider / model / retry. Sample row fixture: `core/test/fixtures/usage-sample.sqlite`. Tests: `core/test/usage-fields.test.js`, `core/test/usage.test.js`. | | `core/src/host/tool-schema.js` | The tool definitions the model sees: its `defineTool` wraps DSH's and completes the compiled schema with `explicitObjectShape`, which fills `properties: {}` and `required: []` on every object node (recursing through `properties`, `items`, `oneOf`/`anyOf`/`allOf`, `$defs`, …). A strict OpenAI-compatible gateway reads an absent member as null and rejects the whole request (`null is not of type "array"`), so one argument-less tool would break every turn; argument validation keeps using DSH's schema unchanged. Every host module registers tools through this wrapper. Tests: `core/test/tool-schema.test.js`. | | `core/src/host/project-state.js` | The `project_state` tool (#1702, #1550 Stage 2.4): the Project State files of the Session working folder (`PROJECT.md`, `ROADMAP.md`, `docs/governance.md`, `decisions/*.md`, `LESSONS.md`), listed with byte size, sha256 content hash and last-modified time, or one file handed over whole up to 200,000 bytes. A caller compares the hash with the copy it already had, so a stale copy is visible instead of assumed. Answers `absent`, `too large` (with the size) or `not a repository` rather than an empty document, refuses any path outside that list, and reads nothing at install time: no prompt, system message or Session context is written by a call. Tests: `core/test/project-state.test.js`. | -| `core/src/host/project-assistant.js` | The Project Assistant (#1712, #1550 Stage 2.5): one hidden Bot and Session per Project, created by `project-create` through the roster and refusing an unknown or archived id, with exactly one 09:00 Asia/Shanghai task on that Session through the host's own `schedule-create` path and removed with the Project by `project-delete` β€” never a cron inside the plugin, and no backfill for a Project made before this slice. Memory lives in `state.projectMemory[id]` (`{decisions, questions, digest}`, behind `rt.projectMemoryOf` / `appendProjectMemory` / `setProjectDigest` and the `project-memory` / `project-memory-write` endpoints) keyed by Project id, so one Project cannot read another's entries. Since #1847 that bound is enforced twice, from one value: `MAX_ENTRY` (600, in `constants.js`) is checked by the assistant writer and again by `appendProjectMemory`, the one write door, and since #1886 both count code points (`[...text].length`), not UTF-16 units, so a 600-character emoji note is stored rather than refused as 1,200 units; the endpoint refuses an empty entry (`A Project memory entry needs text`) and one over the cap (`A Project memory entry is at most 600 characters`) instead of storing a blank row or cutting the text, and a refusal leaves the table and `project.updatedAt` untouched; the panel quotes either sentence, in Chinese under a zh locale (`core/src/client/i18n.js`). `buildDigest` answers, in order, what merged since the previous digest, what closed, what is still in progress and on whom, and the next slice, using only board rows (`source`/`column`/`assignee`/`updatedAt`) and the Project State file list; a day with no change is one line, one day has one note and one `deliver()`, and the second build answers `skipped`. Identity is the install's GitHub connector account (the owner's own on a single-person install): with no connector signed in, or on a multi-member host whose `projects.ownerUserId` is not this host's own user, `digest` refuses and escalates instead of posting as the shared app, and every write carries `Written-by: Project Assistant ()` and `Posted-for: `. Since #1733 the memory and the digest are per (Project, owner) β€” `state.projectMemory[projectId].owners[ownerId]` with the primary owner mirrored on the row, behind `rt.projectMemoryFor(id, ownerId)`, `appendProjectMemory(id, kind, entry)`, `setProjectDigest(id, note, ownerId)` β€” and one `projectBoard` read plus one `listProjectState` read serve one render per owner, so the facts are shared and the personalisation is not; a read outside `ownersOf(project)` is refused naming that boundary. Since #1846 the digest reads the Project's own board, not one member's page: `rt.projectBoard(id, null, { internal: true })` is the one opt-in that skips the per-reader `taskVisible` rule, and the channel builds its options from the zone alone, so no request body can ask for it. Tests: `core/test/project-assistant.test.js`, `core/test/project-digest-authz.test.js`. | +| `core/src/host/project-assistant.js` | The Project Assistant (#1712, #1550 Stage 2.5): one hidden Bot and Session per Project, created by `project-create` through the roster and refusing an unknown or archived id, with exactly one 09:00 Asia/Shanghai task on that Session through the host's own `schedule-create` path and removed with the Project by `project-delete` β€” never a cron inside the plugin, and no backfill for a Project made before this slice. Memory lives in `state.projectMemory[id]` (`{decisions, questions, digest}`, behind `rt.projectMemoryOf` / `appendProjectMemory` / `setProjectDigest` and the `project-memory` / `project-memory-write` endpoints) keyed by Project id, so one Project cannot read another's entries. Since #1847 that bound is enforced twice, from one value: `MAX_ENTRY` (600, in `constants.js`) is checked by the assistant writer and again by `appendProjectMemory`, the one write door, and since #1886 both count code points (`codePoints` in `src/host/text.js`, one helper since #1895), not UTF-16 units, so a 600-character emoji note is stored rather than refused as 1,200 units; the endpoint refuses an empty entry (`A Project memory entry needs text`) and one over the cap (`A Project memory entry is at most 600 characters`) instead of storing a blank row or cutting the text, and a refusal leaves the table and `project.updatedAt` untouched; the panel quotes either sentence, in Chinese under a zh locale (`core/src/client/i18n.js`). `buildDigest` answers, in order, what merged since the previous digest, what closed, what is still in progress and on whom, and the next slice, using only board rows (`source`/`column`/`assignee`/`updatedAt`) and the Project State file list; a day with no change is one line, one day has one note and one `deliver()`, and the second build answers `skipped`. Identity is the install's GitHub connector account (the owner's own on a single-person install): with no connector signed in, or on a multi-member host whose `projects.ownerUserId` is not this host's own user, `digest` refuses and escalates instead of posting as the shared app, and every write carries `Written-by: Project Assistant ()` and `Posted-for: `. Since #1733 the memory and the digest are per (Project, owner) β€” `state.projectMemory[projectId].owners[ownerId]` with the primary owner mirrored on the row, behind `rt.projectMemoryFor(id, ownerId)`, `appendProjectMemory(id, kind, entry)`, `setProjectDigest(id, note, ownerId)` β€” and one `projectBoard` read plus one `listProjectState` read serve one render per owner, so the facts are shared and the personalisation is not; a read outside `ownersOf(project)` is refused naming that boundary. Since #1846 the digest reads the Project's own board, not one member's page: `rt.projectBoard(id, null, { internal: true })` is the one opt-in that skips the per-reader `taskVisible` rule, and the channel builds its options from the zone alone, so no request body can ask for it. Tests: `core/test/project-assistant.test.js`, `core/test/project-digest-authz.test.js`. | | `core/src/host/project-chief.js` | The Project Chief (#1859): every Project gets its own Chief, created with the Project by `project-create` β€” never the user's main Chief acting part-time. `rt.projectChief` has the same shape as `rt.projectAssistant` (`create(projectId)`, `chiefOf(projectId)`, `remove(projectId)`), the record carries `chief: { botId, sessionId, createdAt }` beside `assistant`, and the spawned Bot is named `Project Chief ()`, bound with `bindProject` so it carries `projectId`, and given instructions that name that one Project and forbid answering for any other (another Project's name is never quoted). A host without the seam logs once and answers `undefined`, so `project-create` still creates the Project with `chief: { error }`; `project-delete` retires the Chief through `deleteBot` and clears its binding. Tests: `core/test/projects.test.js`. | | `core/src/host/project-team.js` | The Project's team dossier (#1864), read-only. `rt.projectTeam(projectId)` answers `{ rows }` in the board's order β€” the Project's Chief, its assistant, then `roles.maintainers` / `roles.members` β€” with each row `{ botId, name, role, persona, model, color, createdAt, memory: { head, topics }, skills, rooms }`. `persona` is the Bot's `instructions`; `memory.head` is its own `MEMORY.md` (read through `rt.memoryDigest` in `memory.js`, which also supplies the topic names and never a topic body); `skills` is the attached capabilities the library records as `skill` (a pack contributes its skill children); `rooms` is only the rooms this Project owns. Every string and list is capped (`DOSSIER_LIMITS`: persona and memory head 4000, 64 topics, 64 skills, 32 rooms), a deleted Bot is a `null` row instead of an error for the whole answer, and an unknown Project answers `{ rows: [] }`. The endpoint sits beside `project-read` in `authz.js`'s `TASK_ENDPOINTS`, per the owner decision that persona and memory are shared by the Project's members. Tests: `core/test/project-team.test.js`. | | `core/src/shared/artifact.js` | The typed facts a workflow node takes in and hands out (#1272): `parseArtifactSpec` reads `check:tests-ok` / `issue.ready` / `text:topic` into `{ kind, name }` and refuses a spelling the vocabulary does not have; `artifact(spec, revision, { value })` is one instance, and `REVISION_RULES` says what the revision of each kind is (a verdict its SHA, a check the SHA it ran on, a PR its number and `headRefOid`). `freshness(input, currentRevision)` answers present-and-current separately from present, and a fact nobody could read is `unreadable`, never a match. Tests: `core/test/workflow.test.js`. | diff --git a/docs/subsystems/plugin-runtime.md b/docs/subsystems/plugin-runtime.md index f025edd94..cbbc2ab85 100644 --- a/docs/subsystems/plugin-runtime.md +++ b/docs/subsystems/plugin-runtime.md @@ -122,5 +122,6 @@ FlyBot's plugin API surface on top of the patched core: host hooks, client slots - Changing slots without updating types or client tests. - `flybot.remote.chats.deliver({ handle, text, name?, home?, human?, ownerName? })` (#227): `human: true` is the plugin's server-attested "the friend typed this line themselves" (Bot Social `typed_by_owner`, never derived from text) and `ownerName` the authenticated owner label; the host then draws that turn's reply as the person. Plugins that cannot attest leave both out and the line stays the Bot's. - `flybot.bot.send({ from: { kind: 'remote', handle, home?, typedBy? }, ... })`: `typedBy` is the plugin's server-attested owner name for a line the friend typed (Bot Social `typed_by_owner`); `toRemote` (`src/host/hooks.js`) cleans it and the host stores it as the message source's `remote.typedBy` so the chat draws the person, not the Bot. Never derive it from text; plugins that cannot attest omit it. +- `flybot.bot.send` refuses text over 20,000 characters (`MAX_SEND_TEXT`, `src/host/hooks.js`), and since #1895 that bound counts code points through `codePoints` in `src/host/text.js`, so an emoji is one character and the refusal counts what a reader sees. - A remote sender's line that reaches a local Bot's own chat through `flybot.bot.send` stamps that Bot's `friendLineAt` in the roster (its row turns unread in a window where the chat is not open) and marks the sender's first line `remote.firstContact` (#420, `src/host/delivery.js`). A refused send stamps nothing. Covered by `core/test/flybot.test.js`. - The approval card queue (`FlyBotApprovals`) runs all its hooks on every render, before its empty-queue early return: a hook after it makes React throw when the first hold arrives and no card ever draws (#254; `core/test/overlay-approvals.test.js`). diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index f6e173223..8416a7b15 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -4524,9 +4524,10 @@ diff --git a/src/host/memory.js b/src/host/memory.js import { dirname, join } from 'node:path' import { BlockAssembler } from '@deepseek-ai/dsh-llm' -import { defineTool } from '@deepseek-ai/dsh-tools' +-import { splitRef, unquote } from './text.js' +import { currentParticipants } from './room-membership.js' +import { defineTool } from './tool-schema.js' - import { splitRef, unquote } from './text.js' ++import { codePoints, splitRef, unquote } from './text.js' import { output } from './tools.js' +import { retiredBot } from './store.js' +import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js' @@ -4603,6 +4604,49 @@ diff --git a/src/host/memory.js b/src/host/memory.js return { scope: ownScope(bot.id) } } function writeDenied(scope, botId) { +@@ -509,7 +509,10 @@ export function install(rt) { + const secretEntry = text => (rt.redactSecrets?.(text) ?? text) !== text || looksSecret(text) + function entryProblem(text) { + if (text === '') return 'Write the entry to save.' +- if (text.length > MEMORY_LIMITS.entry) return `An entry holds at most ${MEMORY_LIMITS.entry} characters; this one has ${text.length}. Keep one fact per entry, or save the details as several entries in a topic.` ++ // #1895: one character costs one whatever its UTF-16 width, and the number the refusal ++ // prints is the one a reader would count. ++ const size = codePoints(text) ++ if (size > MEMORY_LIMITS.entry) return `An entry holds at most ${MEMORY_LIMITS.entry} characters; this one has ${size}. Keep one fact per entry, or save the details as several entries in a topic.` + if (secretEntry(text)) return 'This looks like a key, token, password, or ID number, which memory never keeps. Keys go to request_secret.' + return undefined + } +@@ -519,14 +519,17 @@ export function install(rt) { + + function checkSize(file, next, label) { + const limit = limitOf(file) ++ const size = codePoints(next) + // A file already past its size (edited by hand) may still shrink. +- if (next.length <= limit || next.length <= file.content.length) return ++ if (size <= limit || size <= codePoints(file.content)) return + throw new Error(file.topic === undefined +- ? `${capital(label)}'s MEMORY.md would have ${next.length} of its ${limit} characters. Make room first: move details into a topic (remember with topic, then forget them here), merge entries with replace, or forget what no longer matters.` +- : `The topic [[${file.topic}]] in ${label} would have ${next.length} of its ${limit} characters. Put the rest in another topic, or forget old entries.`) ++ ? `${capital(label)}'s MEMORY.md would have ${size} of its ${limit} characters. Make room first: move details into a topic (remember with topic, then forget them here), merge entries with replace, or forget what no longer matters.` ++ : `The topic [[${file.topic}]] in ${label} would have ${size} of its ${limit} characters. Put the rest in another topic, or forget old entries.`) + } + const capital = text => text[0].toUpperCase() + text.slice(1) +- const sizeNote = (file, size) => `${file.file}: ${size} of ${limitOf(file)} characters` ++ // #1895: both the bound and the number it prints count code points, so a file of emoji is ++ // measured the way a reader sees it. ++ const sizeNote = (file, text) => `${file.file}: ${codePoints(text)} of ${limitOf(file)} characters` + const allEntries = files => files.flatMap(file => file.entries.map(entry => ({ ...entry, file }))) + const actorOf = actor => ({ actor: actor.id, actorName: actor.name, ...(actor.auto ? { auto: true } : {}) }) + +@@ -568,7 +568,7 @@ export function install(rt) { + await journal({ op: 'add', scope: scopeKey(scope), file: target.file, ...actorOf(actor), after: line }) + noteNews(scope, actor, 'add', line) + const where = topic === undefined ? MAIN : fresh ? `the new topic [[${topic}]], linked from MEMORY.md` : `topic [[${topic}]]` +- return { changed: true, file: target, message: `Saved to ${label}, in ${where} (${sizeNote(target, next.length)}).` } ++ return { changed: true, file: target, message: `Saved to ${label}, in ${where} (${sizeNote(target, next)}).` } + }) + } + @@ -632,7 +632,7 @@ export function install(rt) { news.delete(botId) return queued(async () => { @@ -4627,7 +4671,13 @@ diff --git a/src/host/memory.js b/src/host/memory.js }, output, async execute(args, exec) { -@@ -861,6 +861,20 @@ export function install(rt) { +@@ -856,22 +856,51 @@ export function install(rt) { + const topic = cleanTopic(args.topic) + for (const { label, files } of loaded) { + const file = files.find(item => item.topic === topic) +- if (file !== undefined) return `Topic [[${topic}]] in ${label} (${sizeNote(file, file.content.length)}):\n${file.content.trim() || '(empty)'}` ++ if (file !== undefined) return `Topic [[${topic}]] in ${label} (${sizeNote(file, file.content)}):\n${file.content.trim() || '(empty)'}` + } const known = loaded.flatMap(({ tag, files }) => files.filter(file => file.topic).map(file => `[[${file.topic}]] (${tag})`)) return `No topic [[${String(args.topic).slice(0, 60)}]] there. Topics: ${known.join(', ') || 'none yet'}.` } @@ -4648,7 +4698,12 @@ diff --git a/src/host/memory.js b/src/host/memory.js const where = (tag, file) => `${tag} Β· ${file.topic === undefined ? MAIN : `[[${file.topic}]]`}` const query = String(args.query ?? '').trim().slice(0, 120) if (query === '') { -@@ -872,6 +872,21 @@ export function install(rt) { + return loaded.map(({ tag, label, files }) => { + const [main, ...topics] = files + return [ +- `${capital(label)}: ${MAIN} has ${main.entries.length} entries (${sizeNote(main, main.content.length)})${topics.length > 0 ? `; topics ${topics.map(file => `[[${file.topic}]] (${file.entries.length})`).join(', ')}` : '; no topics'}.`, ++ `${capital(label)}: ${MAIN} has ${main.entries.length} entries (${sizeNote(main, main.content)})${topics.length > 0 ? `; topics ${topics.map(file => `[[${file.topic}]] (${file.entries.length})`).join(', ')}` : '; no topics'}.`, + ...main.entries.map(entry => `[${where(tag, main)}] ${formatEntry(entry.text, entry.meta).slice(2)}`), ].join('\n') }).join('\n\n') } @@ -4679,7 +4734,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js const own = await modelFor(bot) try { return await oneShot(own, system, text, tag) -@@ -1004,7 +1004,7 @@ export function install(rt) { +@@ -1004,11 +1004,11 @@ export function install(rt) { const prior = summaryFailures.get(bot.id) const failed = prior?.hash === hash && prior.model === bot.model ? prior : undefined const wanted = body.regenerate === true || (body.summarize === true && saved?.hash !== hash && failed === undefined) @@ -4688,6 +4743,11 @@ diff --git a/src/host/memory.js b/src/host/memory.js return { botId: bot.id, entries, +- sizes: { own: own[0].content.length, team: team[0].content.length }, ++ sizes: { own: codePoints(own[0].content), team: codePoints(team[0].content) }, + limits: MEMORY_LIMITS, + summary: entries.length > 0 && saved !== undefined ? { text: saved.text, at: saved.at } : null, + fresh: entries.length === 0 || saved?.hash === hash, @@ -1123,17 +1123,28 @@ export function install(rt) { // What is new in a group, from its log: the member's Session holds the same @@ -4749,6 +4809,15 @@ diff --git a/src/host/memory.js b/src/host/memory.js const live = owner.roomId === undefined ? undefined : rt.roomOf(owner.roomId) const kept = seen?.roomId === owner.roomId ? seen?.room : undefined const member = live?.members?.includes(bot.id) === true +@@ -1256,7 +1256,7 @@ export function install(rt) { + const teamFiles = loadScope(TEAM) + const said = read.filter(line => line.who === 'user').map(line => line.text).join('\n') + const { numbered, left } = reviewEntries(ownFiles, teamFiles, REVIEW_MEMORY_MAX, said) +- const sizes = { own: ownFiles[0].content.length, team: teamFiles[0].content.length } ++ const sizes = { own: codePoints(ownFiles[0].content), team: codePoints(teamFiles[0].content) } + const earlier = batch.earlier.map(line => reviewLine(line.who, line.text, bot)) + const language = writtenIn(read.filter(line => line.who === 'user').map(line => line.text.replace(/^User: /, '')), languageNow()) + const reply = await botShot(bot, reviewSystem(writesTeam(bot.id)), reviewRequest({ bot, numbered, left, sizes, where: batch.where, earlier, fresh: fresh.texts, more, language }), { sessionId: rt.chatOf(bot.id), kind: 'memory' }) @@ -1272,7 +1272,7 @@ export function install(rt) { const badTopic = applied.find(change => change.op === 'add' && typeof change.topic === 'string' && change.topic.trim() !== '' && cleanTopic(change.topic) === undefined) if (badTopic !== undefined) throw new Error(`the model's answer names the topic "${clip(badTopic.topic, 60)}", which cannot be a topic name`) @@ -5364,7 +5433,7 @@ diff --git a/src/host/schedules.js b/src/host/schedules.js +import { createUserMessage } from '@deepseek-ai/dsh-llm' +import { quietHoursOf, inQuietHours, clockIn, quietError } from '../shared/quiet-hours.js' +import { defineTool } from './tool-schema.js' -+import { textOf } from './text.js' ++import { codePoints, textOf } from './text.js' +import { output } from './tools.js' +import { retiredBot } from './store.js' @@ -5465,7 +5534,7 @@ diff --git a/src/host/schedules.js b/src/host/schedules.js } if (typeof ctx.inject === 'function') ctx.inject(['schedule'], attach) else if (ctx.get?.('schedule') !== undefined) attach({ schedule: ctx.get('schedule'), effect: (execute, label) => ctx.effect(execute, label) }) -@@ -89,63 +89,554 @@ export function install(rt) { +@@ -89,63 +89,555 @@ export function install(rt) { return rt.groupOwners.get(sessionId) ?? rt.pastOwners.get(sessionId) } const known = owner => owner !== undefined && (owner.botId === undefined ? state.rooms[owner.roomId] !== undefined : state.bots[owner.botId] !== undefined) @@ -5627,7 +5696,8 @@ diff --git a/src/host/schedules.js b/src/host/schedules.js + const title = String(body.title ?? '').trim() + const prompt = String(body.prompt ?? '').trim() + if (title === '') throw invalid('Give the task a name') -+ if (title.length > MAX_TITLE) throw invalid(`Keep the name under ${MAX_TITLE} characters`) ++ // #1895: the bound counts code points, so a name of emoji is measured as a reader counts it. ++ if (codePoints(title) > MAX_TITLE) throw invalid(`Keep the name under ${MAX_TITLE} characters`) + if (prompt === '') throw invalid('Say what the Bot should do') + return { sessionId: bot ? rt.chatOf(bot.id) : room.id, title, prompt } + } @@ -6065,6 +6135,27 @@ diff --git a/src/host/steward.js b/src/host/steward.js // Context sizes between requests are estimates, so the hard line stays a little // below what the model can actually take. const HARD_SHARE = 0.95 +diff --git a/src/host/text.js b/src/host/text.js +--- a/src/host/text.js ++++ b/src/host/text.js +@@ -23,6 +23,17 @@ export const unquote = text => String(text ?? '').trim().replace(/^["'β€œβ€γ€Œ + + export const escapeRegExp = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + ++// The host bounds this change counts go through it (#1895): one character costs one ++// whatever its UTF-16 width, so an emoji inside a 300-character bound is one character, ++// not two. ++export const codePoints = (text) => { ++ const value = String(text) ++ let n = 0 ++ // A code point is one or two UTF-16 units; step by that width so no array is built. ++ for (let i = 0; i < value.length; n += 1) i += value.codePointAt(i) > 0xffff ? 2 : 1 ++ return n ++} ++ + // The part of a long message around the first match, so a search hit stays visible. + export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => { + if (text.length <= max) return text diff --git a/src/host/time-context.js b/src/host/time-context.js --- a/src/host/time-context.js +++ b/src/host/time-context.js @@ -9969,7 +10060,7 @@ diff --git a/test/memory.test.js b/test/memory.test.js const view = (await call('memory-view', { botId: 'b2' })).value assert.equal(view.error, null) assert.equal(view.summary?.text, 'You like tea.') -@@ -1076,3 +1076,94 @@ test('a memory call names the current part of the Bot chat', async (t) => { +@@ -1076,3 +1076,129 @@ test('a memory call names the current part of the Bot chat', async (t) => { assert.equal(answer.ok, true) assert.equal(requests[0].sessionId, 'b2-part2') }) @@ -10064,6 +10155,41 @@ diff --git a/test/memory.test.js b/test/memory.test.js + assert.match(out, /- \[\[memory\]\] β€” linked/) + assert.match(await tool('recall', { related: 'nothinghere' }, 'b2'), /Nothing in your memory and the team memory links "nothinghere"/) +}) ++ ++// #1895: a character outside the BMP is one character, not two. The entry bound counts ++// code points, and a refusal prints the number a reader would count. ++test('the entry bound counts code points (#1895)', async (t) => { ++ const { tool, stop } = start() ++ t.after(() => stop()) ++ const atCap = 'πŸ˜€'.repeat(MEMORY_LIMITS.entry) ++ assert.match(await tool('remember', { text: atCap }, 'b2'), /^Saved to your memory/, '300 emoji is 300 characters') ++ const over = 'πŸ˜€'.repeat(MEMORY_LIMITS.entry + 1) ++ assert.match(await tool('remember', { text: over }, 'b2'), /at most 300 characters; this one has 301\./, 'the refusal counts code points') ++}) ++ ++// #1895: a MEMORY.md at its 4,000-character size is measured in code points too, and the ++// size note prints what a reader would count β€” not twice the emoji count. ++test('a MEMORY.md size and its note count code points (#1895)', async (t) => { ++ const { tool, read, stop, home } = start() ++ t.after(() => stop()) ++ const dir = join(home, 'memory', 'bots', 'b2') ++ const file = join(dir, 'MEMORY.md') ++ const withEmoji = count => `# Memory\n\n- ${'πŸ˜€'.repeat(count)}\n` ++ mkdirSync(dir, { recursive: true }) ++ ++ // A hand-written file of 3,943 code points (about 7,900 UTF-16 units) takes one more entry. ++ writeFileSync(file, withEmoji(3930)) ++ const savedMessage = await tool('remember', { text: 'One more' }, 'b2') ++ assert.match(savedMessage, /^Saved to your memory/, 'a file under 4,000 code points takes an entry') ++ const saved = [...read('bots', 'b2', 'MEMORY.md')].length ++ const added = saved - [...withEmoji(3930)].length ++ assert.ok(saved <= 4000, String(saved)) ++ assert.match(savedMessage, new RegExp(`MEMORY\\.md: ${saved} of 4000 characters`), 'the note prints the code-point count') ++ // Then one code point more in the file puts the next entry one over the bound, and the ++ // refusal prints that code-point number rather than twice it. ++ writeFileSync(file, withEmoji(4001 - added - 13)) ++ assert.match(await tool('remember', { text: 'One more' }, 'b2'), /MEMORY\.md would have 4001 of its 4000 characters/, 'one code point over is refused with the code-point count') ++}) diff --git a/test/model-menu.test.js b/test/model-menu.test.js --- a/test/model-menu.test.js +++ b/test/model-menu.test.js @@ -10583,7 +10709,7 @@ diff --git a/test/schedules.test.js b/test/schedules.test.js test('a group chat task lives on the group Session and lists under the group', async (t) => { const schedule = scheduleService() const { ctx, home, call } = setup({ schedule }) -@@ -121,3 +121,712 @@ test('a due reminder reaches a group as its prompts', () => { +@@ -121,3 +121,728 @@ test('a due reminder reaches a group as its prompts', () => { assert.equal(reminderText(batch), '[Scheduled task] One\n[Scheduled task] Two') assert.equal(reminderText('plain text'), 'plain text') }) @@ -11296,6 +11422,22 @@ diff --git a/test/schedules.test.js b/test/schedules.test.js + assert.equal(copied.ok, true, JSON.stringify(copied)) + assert.equal([...schedule.tasks.values()].at(-1).sessionId, 'b1') +}) ++ ++// #1895: a task name's 120-character bound counts code points, so a name of emoji is ++// measured the way a reader sees it instead of at double length. ++test('a task name counts code points against its 120-character bound (#1895)', async (t) => { ++ const schedule = scheduleService() ++ const { ctx, home, call } = setup({ schedule }) ++ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) }) ++ const atCap = 'πŸ˜€'.repeat(120) ++ const made = await call('schedule-create', { target: 'm1', title: atCap, prompt: 'Go', when: { kind: 'every', minutes: 5 } }) ++ assert.equal(made.ok, true, JSON.stringify(made)) ++ assert.equal([...schedule.tasks.values()].at(-1).record.title, atCap, 'the whole 120-emoji name reaches DSH') ++ const over = 'πŸ˜€'.repeat(121) ++ const refused = await call('schedule-create', { target: 'm1', title: over, prompt: 'Go', when: { kind: 'every', minutes: 5 } }) ++ assert.equal(refused.ok, false) ++ assert.match(refused.error.message, /Keep the name under 120 characters/) ++}) diff --git a/test/session-identity.test.js b/test/session-identity.test.js --- a/test/session-identity.test.js +++ b/test/session-identity.test.js @@ -11454,6 +11596,29 @@ diff --git a/test/steward.test.js b/test/steward.test.js } // The check line never passes the hard line. assert.equal(linesFor({ window: 100_000, reserve: 0, checkRatio: 0.99 }).check, 95_000) +diff --git a/test/text.test.js b/test/text.test.js +--- a/test/text.test.js ++++ b/test/text.test.js +@@ -1,6 +1,18 @@ + import assert from 'node:assert/strict' + import { test } from 'node:test' +-import { clip, createClock, escapeRegExp, estimateTokens, excerpt, isColor, normalize, splitRef, textOf, unquote } from '../src/host/text.js' ++import { clip, codePoints, createClock, escapeRegExp, estimateTokens, excerpt, isColor, normalize, splitRef, textOf, unquote } from '../src/host/text.js' ++ ++// #1895: the one helper every host limit counts with, so a character outside the BMP is ++// one character and never two. ++test('codePoints counts one character as one whatever its UTF-16 width (#1895)', () => { ++ assert.equal(codePoints(''), 0) ++ assert.equal(codePoints('abc'), 3) ++ assert.equal(codePoints('πŸ˜€'), 1, 'an emoji is one, though its UTF-16 length is 2') ++ assert.equal('πŸ˜€'.length, 2, 'the unit the limits used to count it in') ++ assert.equal(codePoints('aπŸ˜€b'), 3) ++ assert.equal(codePoints('ζ—…θ‘Œ'), 2) ++ assert.equal(codePoints(`a${'πŸ˜€'.repeat(4)}`), 5) ++}) + + test('textOf joins the text blocks only', () => { + assert.equal(textOf([{ type: 'text', text: ' a ' }, { type: 'image' }, { type: 'text', text: 'b' }, { type: 'text', text: 3 }]), 'a \nb') diff --git a/test/tool-cells.test.js b/test/tool-cells.test.js --- a/test/tool-cells.test.js +++ b/test/tool-cells.test.js diff --git a/patches/ds-bot/0002-flybot-api-host.patch b/patches/ds-bot/0002-flybot-api-host.patch index 0827675f3..fd32e5d5f 100644 --- a/patches/ds-bot/0002-flybot-api-host.patch +++ b/patches/ds-bot/0002-flybot-api-host.patch @@ -687,9 +687,10 @@ diff --git a/src/host/store.js b/src/host/store.js -import { LEGACY_THEMES, MAX_EXCHANGES, ROOM_MODES } from './constants.js' +import { LEGACY_THEMES, MAX_ENTRY, MAX_EXCHANGES, ROOM_MODES } from './constants.js' import { homeLock } from './home-lock.js' +-import { currentColor, normalize, unquote } from './text.js' +import { openJournal } from './journal.js' - import { currentColor, normalize, unquote } from './text.js' - ++import { codePoints, currentColor, normalize, unquote } from './text.js' ++ +// The saved team's format version. Version 2 (#47) adds the identity schema: member +// records, a per-Bot owner binding, and the owner's designated Chief. A version-1 file +// is migrated on load (after a one-time `state.json.v1` backup); a file written by a @@ -709,7 +710,7 @@ diff --git a/src/host/store.js b/src/host/store.js +// because every such path imports it rather than growing its own copy. +export const retiredBot = bot => bot?.fields?.['flybot-tree']?.retired === true + -+ + export function install(rt) { - const { ctx, home, statePath } = rt + const { ctx, home, statePath, config = {} } = rt @@ -900,7 +901,7 @@ diff --git a/src/host/store.js b/src/host/store.js // LEGACY: Bots saved with a lab ran on that lab's default model; the model they // last ran on becomes their own, so dropping the lab changes nothing for them. if ('lab' in bot) { -@@ -161,12 +161,478 @@ export function install(rt) { +@@ -161,12 +161,479 @@ export function install(rt) { migrated = true } // Unnamed groups from before names followed their members may still list old names. @@ -1112,8 +1113,9 @@ diff --git a/src/host/store.js b/src/host/store.js + const entryText = projectText(entry.text) + if (entryText === '') throw projectRefusal('A Project memory entry needs text') + // #1886: the bound counts code points, not UTF-16 units (`MAX_ENTRY` says why), so the emoji -+ // note the assistant writer accepts at 600 characters is accepted here too. -+ if ([...entryText].length > MAX_ENTRY) throw projectRefusal(`A Project memory entry is at most ${MAX_ENTRY} characters`) ++ // note the assistant writer accepts at 600 characters is accepted here too. #1895: both ++ // writers count through the one `codePoints` helper. ++ if (codePoints(entryText) > MAX_ENTRY) throw projectRefusal(`A Project memory entry is at most ${MAX_ENTRY} characters`) + const table = memoryTable(project) + const ownerId = projectText(entry.ownerId, 120) || primaryOwner(project) + const record = ownerRow(table, ownerId, project) diff --git a/patches/ds-bot/files/src/host/hooks.js b/patches/ds-bot/files/src/host/hooks.js index 2d8b9e021..96f933d5b 100644 --- a/patches/ds-bot/files/src/host/hooks.js +++ b/patches/ds-bot/files/src/host/hooks.js @@ -23,6 +23,7 @@ import { FEED_EVENTS, installFeed } from './flybot-feed.js' import { createPluginModel } from './plugin-model.js' import { defineTool } from './tool-schema.js' import { retiredBot } from './store.js' +import { codePoints } from './text.js' import { EPHEMERAL_DENIED_TOOLS } from './tools.js' export const FLYBOT_API_VERSION = 1 @@ -728,7 +729,8 @@ export function install(rt) { async function send({ from, to, text, role = 'request', hop = 0, via = SHARED, meta, attachments } = {}) { await rt.load() if (typeof text !== 'string' || text.trim() === '') throw new Error('flybot: send needs text') - if (text.length > MAX_SEND_TEXT) throw new Error(`flybot: a message is at most ${MAX_SEND_TEXT} characters`) + // #1895: the bound counts code points, so a message of emoji is measured as a reader counts it. + if (codePoints(text) > MAX_SEND_TEXT) throw new Error(`flybot: a message is at most ${MAX_SEND_TEXT} characters`) if (role !== 'request' && role !== 'reply') throw new Error(`flybot: unsupported role ${role}`) if (!isRemote(from) && rt.botOf(from) === undefined) throw new Error(`flybot: unknown sender ${from}`) if (!isRemote(to) && rt.botOf(to) === undefined) throw new Error(`flybot: unknown Bot ${to}`) diff --git a/patches/ds-bot/files/src/host/project-assistant.js b/patches/ds-bot/files/src/host/project-assistant.js index ca3995a01..6a37f5867 100644 --- a/patches/ds-bot/files/src/host/project-assistant.js +++ b/patches/ds-bot/files/src/host/project-assistant.js @@ -17,6 +17,7 @@ import { defineTool } from './tool-schema.js' import { MAX_ENTRY } from './constants.js' import { listProjectState } from './project-state.js' +import { codePoints } from './text.js' import { output } from './tools.js' import { inQuietHours, clockIn, quietHoursOf, readQuietHours } from '../shared/quiet-hours.js' @@ -190,7 +191,8 @@ export function install(rt) { if (entry === '') throw refusal('A Project memory entry needs text') // #1886: count code points, the same unit `appendProjectMemory` counts, so the two writers // cannot disagree about which note fits (`String#length` charged an emoji two units). - if ([...entry].length > MAX_ENTRY) throw refusal(`A Project memory entry is at most ${MAX_ENTRY} characters`) + // #1895: both count through the one `codePoints` helper. + if (codePoints(entry) > MAX_ENTRY) throw refusal(`A Project memory entry is at most ${MAX_ENTRY} characters`) const ownerId = text(body.ownerId) || ownersOf(project)[0] || '' if (text(body.ownerId) !== '' && !ownersOf(project).includes(text(body.ownerId))) throw outsideBoundary(project, body.ownerId) rt.appendProjectMemory?.(project.id, kind, { text: entry, at: Date.now(), ownerId, ...(text(body.by) === '' ? {} : { by: text(body.by) }) }) diff --git a/patches/ds-bot/files/test/flybot.test.js b/patches/ds-bot/files/test/flybot.test.js index 557c555db..508581262 100644 --- a/patches/ds-bot/files/test/flybot.test.js +++ b/patches/ds-bot/files/test/flybot.test.js @@ -1037,3 +1037,13 @@ test('#587: list_bots marks a Bot whose send waits on the owner with a count, ne const after = await tool('list_bots', {}, 'a') assert.ok(!after.includes('to approve'), 'a decided hold leaves no tag') }) + +// #1895: a message's 20,000-character bound counts code points, so an emoji message is +// measured the way a reader sees it instead of at double length. +test('a message counts code points against its 20,000-character bound (#1895)', async (t) => { + const { fb, stop } = await start() + t.after(stop) + const atCap = 'πŸ˜€'.repeat(20_000) + assert.equal((await fb.bot.send({ from: 'a', to: 'b', text: atCap })).action, 'allow', '20,000 emoji is 20,000 characters') + await assert.rejects(fb.bot.send({ from: 'a', to: 'b', text: 'πŸ˜€'.repeat(20_001) }), /a message is at most 20000 characters/, 'one code point over is refused') +}) diff --git a/patches/ds-bot/topics.json b/patches/ds-bot/topics.json index e78f10c97..705dba5c6 100644 --- a/patches/ds-bot/topics.json +++ b/patches/ds-bot/topics.json @@ -55,7 +55,9 @@ "src/host/steward.js", "test/steward.test.js", "test/steward-retry.test.js", - "test/i18n.test.js" + "test/i18n.test.js", + "src/host/text.js", + "test/text.test.js" ], "0002-flybot-api-host": [ "index.js",