From 39583fd41d7514b3083fdc322e6045d1adf97669 Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 00:40:28 -0500 Subject: [PATCH 01/13] feat(cms): render and convert lists, emphasis, quotes and links in article bodies One shared Markdown<->Lexical module (cms/src/lib/richText.ts) replaces the duplicated converters. The writer's lists, bold, italic, code, quotes and http(s)/site-relative links now become real Lexical nodes, the public page renders them with an escaped URL allow-list, and the readers emit one line per list item and line break so QA text no longer runs words together. The writer prompt names the supported subset. Co-Authored-By: Claude Sonnet 5.5 --- cms/src/lib/lexicalHtml.ts | 137 +------- cms/src/lib/richText.ts | 491 +++++++++++++++++++++++++++ cms/tests/int/richText.int.spec.ts | 229 +++++++++++++ pipeline/src/generatePrompt.ts | 1 + pipeline/src/richtext.ts | 111 +----- pipeline/test/generatePrompt.test.ts | 10 + pipeline/test/richtext.test.ts | 212 ++++++++++++ 7 files changed, 977 insertions(+), 214 deletions(-) create mode 100644 cms/src/lib/richText.ts create mode 100644 cms/tests/int/richText.int.spec.ts create mode 100644 pipeline/test/richtext.test.ts diff --git a/cms/src/lib/lexicalHtml.ts b/cms/src/lib/lexicalHtml.ts index 4350307e..6e0b6ed2 100644 --- a/cms/src/lib/lexicalHtml.ts +++ b/cms/src/lib/lexicalHtml.ts @@ -1,63 +1,16 @@ import type { Article, Template } from '../payload-types' - -type LexNode = { - type?: string - tag?: string - text?: string - children?: LexNode[] - listType?: string - direction?: string | null - format?: string | number - indent?: number - version?: number - mode?: string - style?: string - detail?: number - textFormat?: number -} +import { + extractHeadings, + lexicalToHtml, + lexicalToMarkdown, + markdownToLexical, +} from './richText' type RichText = NonNullable -function textOf(node: LexNode | undefined): string { - if (!node) return '' - if (typeof node.text === 'string') return node.text - if (Array.isArray(node.children)) return node.children.map(textOf).join('') - return '' -} - -function escapeHtml(s: string): string { - return s - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') -} - -/** Minimal Lexical JSON → React-safe HTML string (headings, paragraphs, lists). */ +/** Lexical JSON → React-safe HTML string. See `richText.ts` for the supported nodes. */ export function lexicalBodyToHtml(body: Article['body']): string { - if (!body?.root?.children?.length) return '' - const parts: string[] = [] - for (const child of body.root.children as LexNode[]) { - const type = child.type - if (type === 'heading') { - const tag = child.tag === 'h1' || child.tag === 'h2' || child.tag === 'h3' ? child.tag : 'h2' - parts.push(`<${tag}>${escapeHtml(textOf(child))}`) - } else if (type === 'paragraph') { - parts.push(`

${escapeHtml(textOf(child))}

`) - } else if (type === 'list') { - const listTag = child.listType === 'number' ? 'ol' : 'ul' - const items = (child.children ?? []) - .map((item) => `
  • ${escapeHtml(textOf(item))}
  • `) - .join('') - parts.push(`<${listTag}>${items}`) - } else if (type === 'quote') { - parts.push(`
    ${escapeHtml(textOf(child))}
    `) - } else { - const t = textOf(child).trim() - if (t) parts.push(`

    ${escapeHtml(t)}

    `) - } - } - return parts.join('\n') + return lexicalToHtml(body) } /** @@ -68,77 +21,19 @@ export function lexicalBodyToHtml(body: Article['body']): string { * section straight into the outline. */ export function lastH2HeadingText(body: Article['body']): string | null { - const children = (body?.root?.children as LexNode[] | undefined) ?? [] - let last: string | null = null - for (const child of children) { - if (child.type === 'heading' && child.tag === 'h2') last = textOf(child).trim() - } - return last + const h2 = extractHeadings(body).filter((heading) => heading.level === 2) + return h2.length ? h2[h2.length - 1].text : null } -/** Lexical → plain lines for config textareas (#/##/### preserved when present). */ +/** + * Lexical → markdown lines for config textareas and prompts (headings, list + * items, quotes and inline marks are kept), one block per line. + */ export function lexicalToPlainText(body: RichText | null | undefined): string { - if (!body?.root?.children?.length) return '' - const lines: string[] = [] - for (const child of body.root.children as LexNode[]) { - const text = textOf(child).trim() - if (!text) continue - if (child.type === 'heading' && typeof child.tag === 'string') { - const level = Number(child.tag.replace('h', '')) || 2 - lines.push(`${'#'.repeat(Math.min(3, Math.max(1, level)))} ${text}`) - } else { - lines.push(text) - } - } - return lines.join('\n') + return lexicalToMarkdown(body, '\n') } /** Plain / light markdown → Lexical (matches pipeline seed shape). */ export function plainTextToLexical(markdown: string): RichText { - const children: LexNode[] = [] - for (const rawLine of markdown.split('\n')) { - const line = rawLine.trim() - if (!line) continue - const match = /^(#{1,3})\s+(.*)$/.exec(line) - const textNode: LexNode = { - detail: 0, - format: 0, - mode: 'normal', - style: '', - text: match ? match[2].trim() : line, - type: 'text', - version: 1, - } - if (match) { - children.push({ - children: [textNode], - direction: 'ltr', - format: '', - indent: 0, - tag: `h${match[1].length}`, - type: 'heading', - version: 1, - }) - } else { - children.push({ - children: [textNode], - direction: 'ltr', - format: '', - indent: 0, - textFormat: 0, - type: 'paragraph', - version: 1, - }) - } - } - return { - root: { - children: children as RichText['root']['children'], - direction: 'ltr', - format: '', - indent: 0, - type: 'root', - version: 1, - }, - } + return markdownToLexical(markdown) as RichText } diff --git a/cms/src/lib/richText.ts b/cms/src/lib/richText.ts new file mode 100644 index 00000000..af0ed849 --- /dev/null +++ b/cms/src/lib/richText.ts @@ -0,0 +1,491 @@ +/** + * The one Markdown <-> Lexical converter, shared by both workspaces. + * + * Dependency-free on purpose: the pipeline imports this file directly, and the + * CMS public page, admin review view and template editor use it too. Keep + * imports out of it. + * + * Writing direction (`markdownToLexical`) handles the subset the writer prompt + * names: `#`..`######` headings, `-`/`*` bullets, `1.` numbered lists, `> ` + * quotes, and inline `**bold**`, `*italic*` / `_italic_`, `` `code` `` and + * `[text](url)` links. Anything else stays literal text. Lists are flat: an + * indented bullet becomes an item of the same list. Backslash escapes are not + * understood. + * + * Reading direction (`lexicalToHtml`, `lexicalToText`, `lexicalToMarkdown`) + * also handles what the admin editor can store that the writer cannot: nested + * lists, underline, strikethrough, line breaks. + * + * Node shapes match what Payload's Lexical editor stores. Text `format` is a + * bit field: 1 bold, 2 italic, 4 strikethrough, 8 underline, 16 code. + */ + +export interface LexicalNode { + type: string + version: number + [key: string]: unknown +} + +export interface LexicalDoc { + root: { + type: 'root' + children: LexicalNode[] + direction: 'ltr' + format: '' + indent: 0 + version: 1 + } + [key: string]: unknown +} + +export interface HeadingInfo { + level: number + text: string +} + +export const FORMAT_BOLD = 1 +export const FORMAT_ITALIC = 2 +export const FORMAT_STRIKETHROUGH = 4 +export const FORMAT_UNDERLINE = 8 +export const FORMAT_CODE = 16 + +// ---------------------------------------------------------------- URLs + +/** + * The URL when it is safe to put in an `href`, else `null`: `http:`, `https:` + * or a site-relative path starting with a single `/`. Whitespace, control + * characters and backslashes are refused, so `javascript:` tricks, `//host` + * (protocol-relative) and `/\host` all fail. + */ +export function safeLinkUrl(raw: unknown): string | null { + if (typeof raw !== 'string') return null + const url = raw.trim() + // eslint-disable-next-line no-control-regex + if (!url || /[\u0000- \u007f-\u009f\\]/.test(url)) return null + if (url.startsWith('/')) return url.startsWith('//') ? null : url + if (!/^https?:\/\/[^/?#]/i.test(url)) return null + try { + const { protocol } = new URL(url) + return protocol === 'http:' || protocol === 'https:' ? url : null + } catch { + return null + } +} + +// ------------------------------------------------- Markdown -> Lexical + +const textNode = (text: string, format: number): LexicalNode => ({ + detail: 0, + format, + mode: 'normal', + style: '', + text, + type: 'text', + version: 1, +}) + +const linkNode = (url: string, children: LexicalNode[]): LexicalNode => ({ + children, + direction: 'ltr', + format: '', + indent: 0, + type: 'link', + version: 3, + fields: { linkType: 'custom', newTab: false, url }, +}) + +const linebreakNode = (): LexicalNode => ({ type: 'linebreak', version: 1 }) + +const LINK_AT_START = /^\[([^[\]]+)\]\(([^()\s]+)\)/ +const isSpace = (ch: string | undefined) => ch === undefined || /\s/.test(ch) +const isWordChar = (ch: string | undefined) => ch !== undefined && /[\p{L}\p{N}]/u.test(ch) + +/** Index of the closing run of `n` `marker` characters at or after `from`, or -1. */ +function findCloser(src: string, marker: string, n: number, from: number): number { + const needle = marker.repeat(n) + for (let j = src.indexOf(needle, from); j !== -1; j = src.indexOf(needle, j + 1)) { + if (isSpace(src[j - 1])) continue + // A run longer than n is a different delimiter (`*` vs `**`); skip it. + if (src[j - 1] === marker || src[j + n] === marker) continue + if (marker === '_' && isWordChar(src[j + n])) continue + return j + } + return -1 +} + +function parseInline(src: string, format = 0, inLink = false): LexicalNode[] { + const out: LexicalNode[] = [] + let buf = '' + const flush = () => { + if (buf) out.push(textNode(buf, format)) + buf = '' + } + let i = 0 + while (i < src.length) { + const ch = src[i] + + if (ch === '`') { + const end = src.indexOf('`', i + 1) + if (end > i + 1) { + flush() + out.push(textNode(src.slice(i + 1, end), format | FORMAT_CODE)) + i = end + 1 + continue + } + } + + if (ch === '[' && !inLink) { + const match = LINK_AT_START.exec(src.slice(i)) + if (match) { + const url = safeLinkUrl(match[2]) + if (url) { + flush() + out.push(linkNode(url, parseInline(match[1], format, true))) + } else { + buf += match[0] // unsafe scheme: keep the markdown as literal text + } + i += match[0].length + continue + } + } + + if (ch === '*' || ch === '_') { + let run = 1 + while (src[i + run] === ch) run++ + const n = Math.min(run, 3) + const opens = !isSpace(src[i + n]) && (ch === '*' || !isWordChar(src[i - 1])) + const close = opens ? findCloser(src, ch, n, i + n) : -1 + if (close > i + n) { + flush() + const bits = n === 1 ? FORMAT_ITALIC : n === 2 ? FORMAT_BOLD : FORMAT_BOLD | FORMAT_ITALIC + out.push(...parseInline(src.slice(i + n, close), format | bits, inLink)) + i = close + n + } else { + buf += ch.repeat(run) // no closer: literal, and do not retry inside the run + i += run + } + continue + } + + buf += ch + i++ + } + flush() + return out +} + +const block = (type: string, children: LexicalNode[], extra: Record = {}) => + ({ + children, + direction: 'ltr', + format: '', + indent: 0, + ...extra, + type, + version: 1, + }) as LexicalNode + +const HEADING = /^(#{1,6})\s+(.+)$/ +const BULLET = /^[-*+]\s+(.+)$/ +// Up to three digits, so a line that opens with a year ("1990. The ...") stays a paragraph. +const NUMBERED = /^(\d{1,3})[.)]\s+(.+)$/ +const QUOTE = /^>\s*(.+)$/ + +/** + * Markdown (the subset described at the top of this file) -> Lexical. + * Each non-blank line is its own block, except that consecutive list lines of + * one kind form one list and consecutive `> ` lines form one quote. + */ +export function markdownToLexical(markdown: string): LexicalDoc { + const children: LexicalNode[] = [] + let list: { listType: 'bullet' | 'number'; start: number; items: LexicalNode[] } | null = null + let quote: LexicalNode[] | null = null + + const closeList = () => { + if (!list) return + children.push( + block('list', list.items, { + listType: list.listType, + start: list.start, + tag: list.listType === 'number' ? 'ol' : 'ul', + }), + ) + list = null + } + const closeQuote = () => { + if (quote) children.push(block('quote', quote)) + quote = null + } + const addItem = (listType: 'bullet' | 'number', start: number, text: string) => { + if (list && list.listType !== listType) closeList() + if (!list) list = { listType, start, items: [] } + list.items.push(block('listitem', parseInline(text), { value: list.start + list.items.length })) + } + + for (const rawLine of markdown.split('\n')) { + const line = rawLine.trim() + if (!line) continue // a blank line does not end a list: "- a", "", "- b" is one list + + const bullet = BULLET.exec(line) + const numbered = bullet ? null : NUMBERED.exec(line) + const quoted = bullet || numbered ? null : QUOTE.exec(line) + + if (!bullet && !numbered) closeList() + if (!quoted) closeQuote() + + if (bullet) { + addItem('bullet', 1, bullet[1]) + } else if (numbered) { + addItem('number', Math.max(1, Number.parseInt(numbered[1], 10)), numbered[2]) + } else if (quoted) { + const inline = parseInline(quoted[1]) + if (quote) quote.push(linebreakNode(), ...inline) + else quote = inline + } else { + const heading = HEADING.exec(line) + if (heading) { + children.push( + block('heading', parseInline(heading[2].trim()), { tag: `h${heading[1].length}` }), + ) + } else { + children.push(block('paragraph', parseInline(line), { textFormat: 0 })) + } + } + } + closeList() + closeQuote() + + return { + root: { + children, + direction: 'ltr', + format: '', + indent: 0, + type: 'root', + version: 1, + }, + } +} + +// ------------------------------------------------- Lexical readers + +type Loose = Record + +const asNode = (value: unknown): Loose | null => + typeof value === 'object' && value !== null ? (value as Loose) : null + +const childrenOf = (node: Loose): Loose[] => + Array.isArray(node.children) + ? node.children.map(asNode).filter((n): n is Loose => n !== null) + : [] + +/** The root's block nodes; `[]` for a missing or malformed body. */ +export function rootChildren(body: unknown): Loose[] { + const root = asNode(asNode(body)?.root) + return root ? childrenOf(root) : [] +} + +const formatOf = (node: Loose): number => (typeof node.format === 'number' ? node.format : 0) + +/** The URL of a `link` / `autolink` node when it has a safe one. */ +function linkUrlOf(node: Loose): string | null { + const fields = asNode(node.fields) + if (fields?.linkType === 'internal') return null + return safeLinkUrl(fields?.url ?? node.url) +} + +/** All text under a node. A `linebreak` is a newline; no markers. */ +export function inlineText(node: unknown): string { + const n = asNode(node) + if (!n) return '' + if (n.type === 'linebreak') return '\n' + if (n.type === 'tab') return '\t' + if (typeof n.text === 'string') return n.text + return childrenOf(n).map(inlineText).join('') +} + +function inlineMarkdown(node: Loose): string { + if (node.type === 'linebreak') return '\n' + if (node.type === 'tab') return '\t' + if (typeof node.text === 'string') { + const format = formatOf(node) + const [, lead = '', core = '', trail = ''] = /^(\s*)([\s\S]*?)(\s*)$/.exec(node.text) ?? [] + if (!core) return node.text + let marked = core + if (format & FORMAT_CODE) marked = `\`${marked}\`` + if (format & FORMAT_BOLD) marked = `**${marked}**` + if (format & FORMAT_ITALIC) marked = `*${marked}*` + return lead + marked + trail + } + const inner = childrenOf(node).map(inlineMarkdown).join('') + if (node.type === 'link' || node.type === 'autolink') { + const url = linkUrlOf(node) + return url && inner.trim() ? `[${inner}](${url})` : inner + } + return inner +} + +const levelOf = (tag: unknown, fallback: number): number => { + const level = typeof tag === 'string' ? Number.parseInt(tag.replace(/^h/, ''), 10) : Number.NaN + return level >= 1 && level <= 6 ? level : fallback +} + +/** Headings in document order, with the level taken from the tag. */ +export function extractHeadings(body: unknown): HeadingInfo[] { + const headings: HeadingInfo[] = [] + for (const child of rootChildren(body)) { + if (child.type !== 'heading' || typeof child.tag !== 'string') continue + const level = Number.parseInt(child.tag.replace(/^h/, ''), 10) + if (!Number.isNaN(level)) headings.push({ level, text: inlineText(child).trim() }) + } + return headings +} + +/** One line per list item, nested items flattened (markdown: indented, prefixed). */ +function listLines(list: Loose, depth: number, markdown: boolean): string[] { + const ordered = list.listType === 'number' + const start = typeof list.start === 'number' && list.start >= 1 ? list.start : 1 + const lines: string[] = [] + let index = 0 + for (const item of childrenOf(list)) { + if (item.type === 'list') { + lines.push(...listLines(item, depth + 1, markdown)) + continue + } + const kids = childrenOf(item) + const text = kids + .filter((kid) => kid.type !== 'list') + .map((kid) => (markdown ? inlineMarkdown(kid) : inlineText(kid))) + .join('') + .replace(/\s*\n\s*/g, ' ') + .trim() + if (text) { + const marker = ordered ? `${start + index}. ` : '- ' + lines.push(markdown ? `${' '.repeat(depth)}${marker}${text}` : text) + index++ + } + for (const nested of kids) { + if (nested.type === 'list') lines.push(...listLines(nested, depth + 1, markdown)) + } + } + return lines +} + +function blockLines(node: Loose, markdown: boolean): string[] { + if (node.type === 'list') return listLines(node, 0, markdown) + const text = (markdown ? childrenOf(node).map(inlineMarkdown).join('') : inlineText(node)).trim() + if (!text) return [] + if (node.type === 'heading' && markdown) { + return [`${'#'.repeat(levelOf(node.tag, 2))} ${text.replace(/\s*\n\s*/g, ' ')}`] + } + if (node.type === 'quote' && markdown) return text.split('\n').map((line) => `> ${line}`) + return [text] +} + +/** + * Plain text for QA, reading grade, banned-phrase scans and similarity: no + * markers, one line per block, one line per list item, a newline for a line + * break. Blocks are joined with a single newline. + */ +export function lexicalToText(body: unknown): string { + return rootChildren(body) + .flatMap((child) => blockLines(child, false)) + .join('\n') +} + +/** + * Markdown for prompts and config textareas: `#` headings, `- ` / `1. ` + * list items, `> ` quotes, `**bold**`, `*italic*`, `` `code` `` and + * `[text](url)`. Blocks are joined with `separator` (a blank line by default). + */ +export function lexicalToMarkdown(body: unknown, separator = '\n\n'): string { + const blocks = rootChildren(body).map((child) => blockLines(child, true).join('\n')) + return blocks.filter((text) => text.length > 0).join(separator) +} + +// ------------------------------------------------- Lexical -> HTML + +function escapeHtml(s: string): string { + return s + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') +} + +function textHtml(node: Loose): string { + const format = formatOf(node) + let html = escapeHtml(String(node.text)) + if (format & FORMAT_CODE) html = `${html}` + if (format & FORMAT_STRIKETHROUGH) html = `${html}` + if (format & FORMAT_UNDERLINE) html = `${html}` + if (format & FORMAT_ITALIC) html = `${html}` + if (format & FORMAT_BOLD) html = `${html}` + return html +} + +function inlineHtml(node: Loose): string { + if (node.type === 'linebreak') return '
    ' + if (node.type === 'tab') return '\t' + if (typeof node.text === 'string') return textHtml(node) + const inner = childrenOf(node).map(inlineHtml).join('') + if (node.type === 'link' || node.type === 'autolink') { + const url = linkUrlOf(node) + if (!url) return inner + const fields = asNode(node.fields) + const target = fields?.newTab === true ? ' target="_blank" rel="noopener noreferrer"' : '' + return `${inner}` + } + return inner +} + +function listHtml(list: Loose): string { + const tag = list.listType === 'number' ? 'ol' : 'ul' + const start = typeof list.start === 'number' && list.start > 1 ? Math.floor(list.start) : 1 + const startAttr = tag === 'ol' && start > 1 ? ` start="${start}"` : '' + const items = childrenOf(list) + .map((item) => { + // A nested list can sit in its own item (the editor) or beside it. + if (item.type === 'list') return `
  • ${listHtml(item)}
  • ` + const kids = childrenOf(item) + const text = kids + .filter((kid) => kid.type !== 'list') + .map(inlineHtml) + .join('') + const nested = kids + .filter((kid) => kid.type === 'list') + .map(listHtml) + .join('') + return `
  • ${text}${nested}
  • ` + }) + .join('') + return `<${tag}${startAttr}>${items}` +} + +/** Lexical JSON -> HTML string. Every text and attribute value is escaped. */ +export function lexicalToHtml(body: unknown): string { + const parts: string[] = [] + for (const child of rootChildren(body)) { + switch (child.type) { + case 'heading': { + const tag = `h${levelOf(child.tag, 2)}` + parts.push(`<${tag}>${childrenOf(child).map(inlineHtml).join('')}`) + break + } + case 'paragraph': + parts.push(`

    ${childrenOf(child).map(inlineHtml).join('')}

    `) + break + case 'list': + parts.push(listHtml(child)) + break + case 'quote': + parts.push(`
    ${childrenOf(child).map(inlineHtml).join('')}
    `) + break + default: { + const text = inlineText(child).trim() + if (text) parts.push(`

    ${escapeHtml(text)}

    `) + } + } + } + return parts.join('\n') +} diff --git a/cms/tests/int/richText.int.spec.ts b/cms/tests/int/richText.int.spec.ts new file mode 100644 index 00000000..ba05791b --- /dev/null +++ b/cms/tests/int/richText.int.spec.ts @@ -0,0 +1,229 @@ +import { randomUUID } from 'node:crypto' +import { getPayload } from 'payload' +import { describe, expect, it } from 'vitest' + +import config from '@/payload.config' +import { lexicalBodyToHtml, lexicalToPlainText, plainTextToLexical } from '@/lib/lexicalHtml' +import { lexicalToHtml, lexicalToMarkdown, markdownToLexical, safeLinkUrl } from '@/lib/richText' + +const html = (md: string) => lexicalToHtml(markdownToLexical(md)) + +describe('lexicalToHtml', () => { + it('renders every construct the writer can produce', () => { + const out = html( + [ + '## Section', + '#### Deep', + 'Plain **bold** *it* `code` [guide](https://example.com/a?x=1&y=2).', + '- one', + '- two', + '3. three', + '4. four', + '> quoted', + ].join('\n'), + ) + expect(out).toBe( + [ + '

    Section

    ', + '

    Deep

    ', + '

    Plain bold it code guide.

    ', + '
    • one
    • two
    ', + '
    1. three
    2. four
    ', + '
    quoted
    ', + ].join('\n'), + ) + }) + + it('renders editor-authored formats, line breaks and nested lists', () => { + const t = (text: string, format = 0) => ({ type: 'text', version: 1, text, format }) + const body = { + root: { + type: 'root', + version: 1, + children: [ + { + type: 'paragraph', + version: 1, + children: [t('a', 8), t('b', 4), t('c', 1 | 2), { type: 'linebreak', version: 1 }, t('d')], + }, + { + type: 'list', + version: 1, + listType: 'bullet', + children: [ + { + type: 'listitem', + version: 1, + children: [ + t('outer'), + { + type: 'list', + version: 1, + listType: 'number', + children: [{ type: 'listitem', version: 1, children: [t('inner')] }], + }, + ], + }, + ], + }, + { + type: 'paragraph', + version: 1, + children: [ + { + type: 'link', + version: 3, + fields: { linkType: 'custom', newTab: true, url: 'https://example.com' }, + children: [t('out')], + }, + ], + }, + ], + }, + } + expect(lexicalToHtml(body)).toBe( + [ + '

    abc
    d

    ', + '
    • outer
      1. inner
    ', + '

    out

    ', + ].join('\n'), + ) + }) + + it('keeps old paragraph-only bodies and the lexicalBodyToHtml signature as before', () => { + expect(lexicalBodyToHtml(plainTextToLexical('## Intro\nSome & "more".'))).toBe( + '

    Intro

    \n

    Some <text> & "more".

    ', + ) + expect(lexicalBodyToHtml(null)).toBe('') + expect(lexicalBodyToHtml({ root: { children: [] } } as never)).toBe('') + }) + + it('escapes hostile text everywhere', () => { + const out = html( + '## \n **x**\n- \n> "q" & ', + ) + expect(out).not.toMatch(/|/) + expect(out).toContain('<script>alert(1)</script>') + expect(out).toContain('<b>x</b>') + expect(out).toContain('
    "q" & <i>
    ') + }) + + it('never emits a link for an unsafe URL, however it got into the body', () => { + const hostile = [ + 'javascript:alert(1)', + ' JaVaScRiPt:alert(1)', + 'java\nscript:alert(1)', + 'data:text/html;base64,AAAA', + 'vbscript:x', + '//evil.example', + '/\\evil.example', + "https://ok.example/ onclick=alert(1)", + ] + for (const url of hostile) { + const body = { + root: { + type: 'root', + version: 1, + children: [ + { + type: 'paragraph', + version: 1, + children: [ + { + type: 'link', + version: 3, + fields: { linkType: 'custom', newTab: false, url }, + children: [{ type: 'text', version: 1, text: 'label', format: 0 }], + }, + ], + }, + ], + }, + } + expect(lexicalToHtml(body), url).toBe('

    label

    ') + expect(safeLinkUrl(url), url).toBeNull() + } + }) + + it('escapes a double quote that parses as part of a safe URL', () => { + const out = html('[x](https://ok.example/"onmouseover="alert)') + expect(out).toBe('

    x

    ') + expect(html('[x](https://ok.example/a"b)')).toBe( + '

    x

    ', + ) + }) + + it('renders an internal (document) link as its text', () => { + const body = { + root: { + type: 'root', + version: 1, + children: [ + { + type: 'paragraph', + version: 1, + children: [ + { + type: 'link', + version: 3, + fields: { linkType: 'internal', url: '/should-not-be-used', doc: { value: 1 } }, + children: [{ type: 'text', version: 1, text: 'doc', format: 0 }], + }, + ], + }, + ], + }, + } + expect(lexicalToHtml(body)).toBe('

    doc

    ') + }) +}) + +describe('template textarea round trip', () => { + it('keeps lists, marks and links when a template outline is read and saved again', () => { + const text = '## Intro\n- one\n- **two**\n1. a\n2. [b](/x)\nAfter.' + const body = plainTextToLexical(text) + expect(lexicalToPlainText(body)).toBe(text) + expect(plainTextToLexical(lexicalToPlainText(body))).toEqual(body) + expect(lexicalToMarkdown(body, '\n')).toBe(text) + }) +}) + +describe('payload round trip', () => { + it('saves a converted body through payload.update and reads it back unchanged', async () => { + const payload = await getPayload({ config: await config }) + const body = markdownToLexical( + [ + '## Section', + '#### Deep', + 'Text with **bold**, *italic*, `code` and a [link](https://example.com/page).', + '- first', + '- second **b**', + '1. one', + '2. two', + '> a quote', + '> second line', + ].join('\n'), + ) + const article = await payload.create({ + collection: 'articles', + data: { keyword: `rich text ${randomUUID()}`, title: 'Rich text' }, + }) + // Articles are not deleted: their audit rows are append-only. + { + await payload.update({ + collection: 'articles', + id: article.id, + data: { body: body as never }, + }) + const read = await payload.findByID({ collection: 'articles', id: article.id }) + const stored = read.body as typeof body + expect(lexicalToHtml(stored)).toBe(lexicalToHtml(body)) + expect(lexicalToMarkdown(stored)).toBe(lexicalToMarkdown(body)) + const types = stored.root.children.map((c) => c.type) + expect(types).toEqual(['heading', 'heading', 'paragraph', 'list', 'list', 'quote']) + const list = stored.root.children[3] as Record + expect(list).toMatchObject({ listType: 'bullet', tag: 'ul', start: 1 }) + expect(lexicalToHtml(stored)).toContain('link') + } + }) +}) diff --git a/pipeline/src/generatePrompt.ts b/pipeline/src/generatePrompt.ts index 7f1fbca1..e6de57bb 100644 --- a/pipeline/src/generatePrompt.ts +++ b/pipeline/src/generatePrompt.ts @@ -228,6 +228,7 @@ export function buildPrompt( `# Output`, `Return a JSON object with exactly these keys: title, slug, titleTag, metaDescription, ogTitle, ogDescription, ogImage, faqItems (array of {question, answer}), bodyMarkdown.`, `bodyMarkdown uses ## for sections and ### for subsections, never # (the title is the page H1). Respect the SEO spec limits and the outline's section headings.`, + 'In bodyMarkdown use only this markdown, because it is all the page renders: paragraphs (one per line, blank line between), ## and ### headings, bullet lists (lines starting with "- "), numbered lists ("1. "), quotes ("> "), **bold**, *italic*, `code`, and links as [descriptive anchor](https://example.com) with an http(s) or site-relative / URL. No tables, images, HTML, horizontal rules or nested lists; they would print as raw text.', ...(brandVoice ? ['Every field must follow the brand voice, not only bodyMarkdown.'] : []), // Only when there is a bank to cite. Asking for refs a workspace has no // entries for teaches the model to invent them, and an invented ref is a diff --git a/pipeline/src/richtext.ts b/pipeline/src/richtext.ts index f7db494e..de39df35 100644 --- a/pipeline/src/richtext.ts +++ b/pipeline/src/richtext.ts @@ -1,109 +1,34 @@ import type { Article } from '../../cms/src/payload-types' +import { + extractHeadings as extractHeadingsOf, + lexicalToMarkdown as lexicalToMarkdownOf, + lexicalToText, + markdownToLexical as markdownToLexicalOf, + type HeadingInfo, +} from '../../cms/src/lib/richText' + +// The converter lives in cms/src/lib/richText.ts so the pipeline and the CMS +// read and write exactly the same Lexical shapes. +export type { HeadingInfo } export type RichText = NonNullable export type RichTextNode = RichText['root']['children'][number] -export interface HeadingInfo { - level: number - text: string -} - -const textNode = (text: string): RichTextNode => ({ - detail: 0, - format: 0, - mode: 'normal', - style: '', - text, - type: 'text', - version: 1, -}) - -const headingNode = (tag: 'h1' | 'h2' | 'h3', text: string): RichTextNode => ({ - children: [textNode(text)], - direction: 'ltr', - format: '', - indent: 0, - tag, - type: 'heading', - version: 1, -}) - -const paragraphNode = (text: string): RichTextNode => ({ - children: [textNode(text)], - direction: 'ltr', - format: '', - indent: 0, - textFormat: 0, - type: 'paragraph', - version: 1, -}) - -/** Minimal markdown -> Lexical: #/##/### become heading nodes, everything else paragraphs. */ +/** Markdown (headings, lists, quotes, bold/italic/code, links) -> Lexical. */ export function markdownToLexical(markdown: string): RichText { - const children: RichTextNode[] = [] - for (const rawLine of markdown.split('\n')) { - const line = rawLine.trim() - if (!line) continue - const match = /^(#{1,3})\s+(.*)$/.exec(line) - if (match) { - const tag = `h${match[1].length}` as 'h1' | 'h2' | 'h3' - children.push(headingNode(tag, match[2].trim())) - } else { - children.push(paragraphNode(line)) - } - } - return { - root: { - children, - direction: 'ltr', - format: '', - indent: 0, - type: 'root', - version: 1, - }, - } -} - -function nodeText(node: unknown): string { - if (typeof node !== 'object' || node === null) return '' - const record = node as Record - if (typeof record.text === 'string') return record.text - if (Array.isArray(record.children)) { - return record.children.map(nodeText).join('') - } - return '' + return markdownToLexicalOf(markdown) as RichText } export function extractHeadings(body: RichText): HeadingInfo[] { - const headings: HeadingInfo[] = [] - for (const child of body.root.children) { - if (child.type === 'heading' && typeof child.tag === 'string') { - const level = Number.parseInt(child.tag.replace(/^h/, ''), 10) - if (!Number.isNaN(level)) headings.push({ level, text: nodeText(child).trim() }) - } - } - return headings + return extractHeadingsOf(body) } -/** Flatten rich text to markdown-ish lines, keeping heading markers (used for prompts). */ +/** Flatten rich text to markdown, keeping headings, list markers and links (used for prompts). */ export function lexicalToMarkdown(body: RichText): string { - return body.root.children - .map((child) => { - const text = nodeText(child).trim() - if (!text) return '' - if (child.type === 'heading' && typeof child.tag === 'string') { - const level = Number.parseInt(child.tag.replace(/^h/, ''), 10) || 2 - return `${'#'.repeat(level)} ${text}` - } - return text - }) - .filter((line) => line.length > 0) - .join('\n\n') + return lexicalToMarkdownOf(body) } +/** Plain text, one line per block and per list item, no markers. */ export function lexicalToPlainText(body: RichText): string { - return body.root.children - .map((child) => nodeText(child).trim()) - .filter((t) => t.length > 0) - .join('\n') + return lexicalToText(body) } diff --git a/pipeline/test/generatePrompt.test.ts b/pipeline/test/generatePrompt.test.ts index edab2b03..cba73183 100644 --- a/pipeline/test/generatePrompt.test.ts +++ b/pipeline/test/generatePrompt.test.ts @@ -437,3 +437,13 @@ test('audienceToPrompt adds no scope line when the brand voice describes no read assert.match(block, /^# Audience: Café owners \(primary ICP\)\nConfidence tags say/) assert.equal(audienceToPrompt(BRAND_VOICE_FIXTURE, null), '') }) + +test('buildPrompt names the markdown subset the page renders', () => { + const prompt = buildPrompt(makeArticle(baseResearch), template, null, emptyTenantContext()) + const output = prompt.slice(prompt.indexOf('# Output')) + assert.match(output, /bullet lists/) + assert.match(output, /numbered lists/) + assert.match(output, /\*\*bold\*\*/) + assert.match(output, /\[descriptive anchor\]\(https:\/\/example\.com\)/) + assert.match(output, /No tables, images, HTML/) +}) diff --git a/pipeline/test/richtext.test.ts b/pipeline/test/richtext.test.ts new file mode 100644 index 00000000..8edc97ce --- /dev/null +++ b/pipeline/test/richtext.test.ts @@ -0,0 +1,212 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + extractHeadings, + lexicalToMarkdown, + lexicalToPlainText, + markdownToLexical, + type RichText, +} from '../src/richtext' + +type Node = Record & { children?: Node[] } +const blocks = (md: string) => markdownToLexical(md).root.children as Node[] +const texts = (node: Node) => (node.children ?? []).map((c) => [c.text, c.format]) + +test('headings h1..h6 become heading nodes; seven hashes stay a paragraph', () => { + const nodes = blocks('# a\n## b\n### c\n#### d\n##### e\n###### f\n####### g') + assert.deepEqual( + nodes.map((n) => [n.type, n.tag]), + [ + ['heading', 'h1'], + ['heading', 'h2'], + ['heading', 'h3'], + ['heading', 'h4'], + ['heading', 'h5'], + ['heading', 'h6'], + ['paragraph', undefined], + ], + ) +}) + +test('consecutive bullets form one list in the node shape Payload stores', () => { + const [list, ...rest] = blocks('- one\n* two\n+ three') + assert.equal(rest.length, 0) + assert.equal(list.type, 'list') + assert.equal(list.listType, 'bullet') + assert.equal(list.tag, 'ul') + assert.equal(list.start, 1) + assert.deepEqual( + list.children?.map((i) => [i.type, i.value, (i.children?.[0] as Node).text]), + [ + ['listitem', 1, 'one'], + ['listitem', 2, 'two'], + ['listitem', 3, 'three'], + ], + ) +}) + +test('numbered lists keep their start; a blank line does not split a list; a kind change does', () => { + const nodes = blocks('3. c\n\n4. d\n- x') + assert.equal(nodes.length, 2) + assert.equal(nodes[0].listType, 'number') + assert.equal(nodes[0].tag, 'ol') + assert.equal(nodes[0].start, 3) + assert.deepEqual(nodes[0].children?.map((i) => i.value), [3, 4]) + assert.equal(nodes[1].listType, 'bullet') +}) + +test('a paragraph between lists ends the first list', () => { + assert.deepEqual( + blocks('- a\ntext\n- b').map((n) => n.type), + ['list', 'paragraph', 'list'], + ) +}) + +test('lines that only look like list or heading markers stay paragraphs', () => { + for (const line of ['-', '- ', '*italic* start', '**Bold** start', '1990. A good year', '#nospace', '---']) { + const [node] = blocks(line) + assert.equal(node.type, 'paragraph', line) + } +}) + +test('consecutive quote lines form one quote with a line break', () => { + const [quote, ...rest] = blocks('> first\n> second\nafter') + assert.equal(rest.length, 1) + assert.equal(quote.type, 'quote') + assert.deepEqual( + quote.children?.map((c) => c.type), + ['text', 'linebreak', 'text'], + ) +}) + +test('inline bold, italic, code and nested marks use the format bit field', () => { + const [p] = blocks('a **b** *c* _d_ `e` ***f*** **g *h* i**') + assert.deepEqual(texts(p), [ + ['a ', 0], + ['b', 1], + [' ', 0], + ['c', 2], + [' ', 0], + ['d', 2], + [' ', 0], + ['e', 16], + [' ', 0], + ['f', 3], + [' ', 0], + ['g ', 1], + ['h', 3], + [' i', 1], + ]) +}) + +test('unmatched or intraword markers stay literal', () => { + const [p] = blocks('2 * 3 * 4, snake_case_name, **open, a_b_ and [E3] ref') + assert.deepEqual( + (p.children ?? []).map((c) => c.format), + [0], + ) + assert.equal((p.children?.[0] as Node).text, '2 * 3 * 4, snake_case_name, **open, a_b_ and [E3] ref') +}) + +test('links keep http, https and site-relative URLs in a custom link node', () => { + const [p] = blocks('See [the guide](https://example.com/a?b=1) and [home](/articles/x) now') + const link = (p.children ?? []).filter((c) => c.type === 'link') + assert.equal(link.length, 2) + assert.equal(link[0].version, 3) + assert.deepEqual(link[0].fields, { linkType: 'custom', newTab: false, url: 'https://example.com/a?b=1' }) + assert.deepEqual((link[1].fields as Node).url, '/articles/x') + assert.equal((link[0].children?.[0] as Node).text, 'the guide') +}) + +test('links with any other URL stay literal text', () => { + for (const url of [ + 'javascript:alert(1)', + 'JaVaScRiPt:alert(1)', + 'data:text/html,hi', + 'mailto:a@b.co', + '//evil.example/x', + '/\\evil.example', + 'ftp://example.com', + 'example.com', + 'https://', + ]) { + const [p] = blocks(`go [here](${url}) now`) + assert.ok(!(p.children ?? []).some((c) => c.type === 'link'), url) + assert.equal((p.children?.[0] as Node).text, `go [here](${url}) now`, url) + } +}) + +test('bold and code inside link text and headings are kept', () => { + const [h, p] = blocks('## A **bold** heading\n[**x** and `y`](/z)') + assert.deepEqual(texts(h), [['A ', 0], ['bold', 1], [' heading', 0]]) + const link = p.children?.[0] as Node + assert.deepEqual(texts(link), [['x', 1], [' and ', 0], ['y', 16]]) +}) + +test('headings read back with plain text, ignoring inline marks', () => { + const body = markdownToLexical('## A **bold** [link](/x)\n#### Deep') + assert.deepEqual(extractHeadings(body as RichText), [ + { level: 2, text: 'A bold link' }, + { level: 4, text: 'Deep' }, + ]) +}) + +test('a skipped level from an h4 is now visible to the heading checks', () => { + const body = markdownToLexical('## A\n#### B') as RichText + assert.deepEqual(extractHeadings(body).map((h) => h.level), [2, 4]) +}) + +test('lexicalToMarkdown writes lists, quotes, marks and links back', () => { + const md = [ + '## Heading', + 'A **bold** and *it* with `code` and [a link](https://example.com/x).', + '- one', + '- two **b**', + '> quoted', + '1. first', + '2. second', + ].join('\n') + const out = lexicalToMarkdown(markdownToLexical(md) as RichText) + assert.equal(out, md.split('\n').join('\n\n').replace('- one\n\n- two', '- one\n- two').replace('1. first\n\n2.', '1. first\n2.')) + // Round trip: reading the markdown again yields the same document. + assert.deepEqual(markdownToLexical(out), markdownToLexical(md)) +}) + +test('lexicalToPlainText puts each list item and line break on its own line, with no markers', () => { + const body = markdownToLexical('Intro\n- alpha **beta**\n- gamma\n> q1\n> q2\n1. one\n2. two') as RichText + assert.equal(lexicalToPlainText(body), 'Intro\nalpha beta\ngamma\nq1\nq2\none\ntwo') +}) + +test('editor-authored lists, nesting and line breaks do not run words together', () => { + const t = (text: string, format = 0) => ({ type: 'text', version: 1, text, format }) + const item = (...children: unknown[]) => ({ type: 'listitem', version: 1, children }) + const body = { + root: { + type: 'root', + version: 1, + children: [ + { + type: 'list', + version: 1, + listType: 'number', + start: 1, + children: [ + item(t('alpha')), + item({ type: 'list', version: 1, listType: 'bullet', children: [item(t('nested'))] }), + item(t('beta')), + ], + }, + { type: 'paragraph', version: 1, children: [t('line one'), { type: 'linebreak', version: 1 }, t('line two')] }, + ], + }, + } as unknown as RichText + assert.equal(lexicalToPlainText(body), 'alpha\nnested\nbeta\nline one\nline two') + assert.equal(lexicalToMarkdown(body), '1. alpha\n - nested\n2. beta\n\nline one\nline two') +}) + +test('old bodies (one paragraph per line, raw text) read exactly as before', () => { + const body = markdownToLexical('## Intro\nPlain text here.\nMore text.') as RichText + assert.equal(lexicalToMarkdown(body), '## Intro\n\nPlain text here.\n\nMore text.') + assert.equal(lexicalToPlainText(body), 'Intro\nPlain text here.\nMore text.') +}) From 34009473f484fc5f3273d07658cc902332e26aae Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 00:53:42 -0500 Subject: [PATCH 02/13] feat(cms): style lists, links, quotes and code in the reader page Also fixes the rich-text round-trip test setup and a stale lint directive. Co-Authored-By: Claude Sonnet 5.5 --- .../(frontend)/articles/[slug]/article.css | 48 +++++++++++++++++++ cms/src/lib/richText.ts | 1 - cms/tests/int/richText.int.spec.ts | 2 +- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/cms/src/app/(frontend)/articles/[slug]/article.css b/cms/src/app/(frontend)/articles/[slug]/article.css index 402482b2..940d1d98 100644 --- a/cms/src/app/(frontend)/articles/[slug]/article.css +++ b/cms/src/app/(frontend)/articles/[slug]/article.css @@ -86,6 +86,14 @@ color: var(--ink-secondary); } +.datum-public__body h4, +.datum-public__body h5, +.datum-public__body h6 { + font-size: 16px; + font-weight: 600; + margin: 20px 0 6px; +} + .datum-public__body ul, .datum-public__body ol { margin: 0 0 12px; @@ -93,6 +101,46 @@ color: var(--ink-secondary); } +.datum-public__body li { + margin: 0 0 4px; +} + +.datum-public__body li > ul, +.datum-public__body li > ol { + margin: 4px 0 0; +} + +.datum-public__body strong { + font-weight: 600; + color: var(--ink); +} + +.datum-public__body a { + color: var(--primary); + text-decoration: underline; + text-underline-offset: 2px; +} + +.datum-public__body a:hover { + text-decoration-thickness: 2px; +} + +.datum-public__body blockquote { + margin: 0 0 12px; + padding: 2px 0 2px 16px; + border-left: 3px solid var(--hairline); + color: var(--ink-muted); +} + +.datum-public__body code { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.9em; + padding: 1px 5px; + border-radius: 4px; + background: var(--canvas); + border: 1px solid var(--hairline); +} + .datum-public__faq { margin-top: 40px; } diff --git a/cms/src/lib/richText.ts b/cms/src/lib/richText.ts index af0ed849..f893772a 100644 --- a/cms/src/lib/richText.ts +++ b/cms/src/lib/richText.ts @@ -60,7 +60,6 @@ export const FORMAT_CODE = 16 export function safeLinkUrl(raw: unknown): string | null { if (typeof raw !== 'string') return null const url = raw.trim() - // eslint-disable-next-line no-control-regex if (!url || /[\u0000- \u007f-\u009f\\]/.test(url)) return null if (url.startsWith('/')) return url.startsWith('//') ? null : url if (!/^https?:\/\/[^/?#]/i.test(url)) return null diff --git a/cms/tests/int/richText.int.spec.ts b/cms/tests/int/richText.int.spec.ts index ba05791b..dcb44d31 100644 --- a/cms/tests/int/richText.int.spec.ts +++ b/cms/tests/int/richText.int.spec.ts @@ -206,7 +206,7 @@ describe('payload round trip', () => { ) const article = await payload.create({ collection: 'articles', - data: { keyword: `rich text ${randomUUID()}`, title: 'Rich text' }, + data: { keyword: `rich text ${randomUUID()}`, title: 'Rich text', status: 'published' }, }) // Articles are not deleted: their audit rows are append-only. { From 8e011850b7b2f5fd84aacf6f114d78a60d2edebf Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 01:10:01 -0500 Subject: [PATCH 03/13] fix(cms): bound the emphasis scan and keep plain text free of markdown A line of unclosed `*` or `_` openers rescanned the rest of the line once per opener: quadratic, reachable from a template save or the writer's output (120 KB took 18 s). Remember where a closer search already failed. The CMS `lexicalToPlainText` returns plain words again (reading time and the banned-phrase scan read it); the template textareas use the new `lexicalToMarkdownText`. Co-Authored-By: Claude Opus 5.5 --- cms/src/components/ops/templateTypes.ts | 6 ++--- cms/src/lib/lexicalHtml.ts | 15 ++++++++++-- cms/src/lib/richText.ts | 24 ++++++++++++++++--- cms/tests/int/richText.int.spec.ts | 31 ++++++++++++++++++++++--- 4 files changed, 65 insertions(+), 11 deletions(-) diff --git a/cms/src/components/ops/templateTypes.ts b/cms/src/components/ops/templateTypes.ts index f87931ef..b01a9666 100644 --- a/cms/src/components/ops/templateTypes.ts +++ b/cms/src/components/ops/templateTypes.ts @@ -1,4 +1,4 @@ -import { lexicalToPlainText } from '../../lib/lexicalHtml' +import { lexicalToMarkdownText } from '../../lib/lexicalHtml' import { type CompanyMentions, companyMentionsOf } from '../../lib/tenant/companyMentions' import type { Template } from '../../payload-types' @@ -31,8 +31,8 @@ export function toTemplateDTO(doc: Template): TemplateConfigDTO { name: doc.name, updatedAt: doc.updatedAt, companyMentions: companyMentionsOf(doc), - outline: lexicalToPlainText(doc.outline), - example: lexicalToPlainText(doc.example), + outline: lexicalToMarkdownText(doc.outline), + example: lexicalToMarkdownText(doc.example), dos: (doc.dos ?? []).map((d) => d.text), donts: (doc.donts ?? []).map((d) => d.text), requiredSections: (doc.requiredSections ?? []).map((s) => s.heading), diff --git a/cms/src/lib/lexicalHtml.ts b/cms/src/lib/lexicalHtml.ts index 6e0b6ed2..92b9d528 100644 --- a/cms/src/lib/lexicalHtml.ts +++ b/cms/src/lib/lexicalHtml.ts @@ -3,6 +3,7 @@ import { extractHeadings, lexicalToHtml, lexicalToMarkdown, + lexicalToText, markdownToLexical, } from './richText' @@ -26,10 +27,20 @@ export function lastH2HeadingText(body: Article['body']): string | null { } /** - * Lexical → markdown lines for config textareas and prompts (headings, list - * items, quotes and inline marks are kept), one block per line. + * Lexical → plain text, one line per block and per list item, no markers. For + * anything that reads the words (reading time, phrase matching); the template + * textareas use `lexicalToMarkdownText`. */ export function lexicalToPlainText(body: RichText | null | undefined): string { + return lexicalToText(body) +} + +/** + * Lexical → markdown lines for config textareas (headings, list items, quotes + * and inline marks are kept), one block per line, so `plainTextToLexical` reads + * the same content back. + */ +export function lexicalToMarkdownText(body: RichText | null | undefined): string { return lexicalToMarkdown(body, '\n') } diff --git a/cms/src/lib/richText.ts b/cms/src/lib/richText.ts index f893772a..45dba2f1 100644 --- a/cms/src/lib/richText.ts +++ b/cms/src/lib/richText.ts @@ -99,8 +99,24 @@ const LINK_AT_START = /^\[([^[\]]+)\]\(([^()\s]+)\)/ const isSpace = (ch: string | undefined) => ch === undefined || /\s/.test(ch) const isWordChar = (ch: string | undefined) => ch !== undefined && /[\p{L}\p{N}]/u.test(ch) -/** Index of the closing run of `n` `marker` characters at or after `from`, or -1. */ -function findCloser(src: string, marker: string, n: number, from: number): number { +/** + * Index of the closing run of `n` `marker` characters at or after `from`, or -1. + * + * `misses` remembers, per marker and run length, the lowest `from` that found + * nothing. Whether a position is a valid closer does not depend on `from`, so + * a later search that starts at or after it cannot succeed either. Without it, + * a line of n unclosed openers rescans the rest of the line n times. + */ +function findCloser( + src: string, + marker: string, + n: number, + from: number, + misses: Map, +): number { + const key = marker + n + const missedFrom = misses.get(key) + if (missedFrom !== undefined && from >= missedFrom) return -1 const needle = marker.repeat(n) for (let j = src.indexOf(needle, from); j !== -1; j = src.indexOf(needle, j + 1)) { if (isSpace(src[j - 1])) continue @@ -109,11 +125,13 @@ function findCloser(src: string, marker: string, n: number, from: number): numbe if (marker === '_' && isWordChar(src[j + n])) continue return j } + misses.set(key, Math.min(from, missedFrom ?? from)) return -1 } function parseInline(src: string, format = 0, inLink = false): LexicalNode[] { const out: LexicalNode[] = [] + const misses = new Map() let buf = '' const flush = () => { if (buf) out.push(textNode(buf, format)) @@ -153,7 +171,7 @@ function parseInline(src: string, format = 0, inLink = false): LexicalNode[] { while (src[i + run] === ch) run++ const n = Math.min(run, 3) const opens = !isSpace(src[i + n]) && (ch === '*' || !isWordChar(src[i - 1])) - const close = opens ? findCloser(src, ch, n, i + n) : -1 + const close = opens ? findCloser(src, ch, n, i + n, misses) : -1 if (close > i + n) { flush() const bits = n === 1 ? FORMAT_ITALIC : n === 2 ? FORMAT_BOLD : FORMAT_BOLD | FORMAT_ITALIC diff --git a/cms/tests/int/richText.int.spec.ts b/cms/tests/int/richText.int.spec.ts index dcb44d31..d33c1746 100644 --- a/cms/tests/int/richText.int.spec.ts +++ b/cms/tests/int/richText.int.spec.ts @@ -3,7 +3,12 @@ import { getPayload } from 'payload' import { describe, expect, it } from 'vitest' import config from '@/payload.config' -import { lexicalBodyToHtml, lexicalToPlainText, plainTextToLexical } from '@/lib/lexicalHtml' +import { + lexicalBodyToHtml, + lexicalToMarkdownText, + lexicalToPlainText, + plainTextToLexical, +} from '@/lib/lexicalHtml' import { lexicalToHtml, lexicalToMarkdown, markdownToLexical, safeLinkUrl } from '@/lib/richText' const html = (md: string) => lexicalToHtml(markdownToLexical(md)) @@ -182,10 +187,30 @@ describe('template textarea round trip', () => { it('keeps lists, marks and links when a template outline is read and saved again', () => { const text = '## Intro\n- one\n- **two**\n1. a\n2. [b](/x)\nAfter.' const body = plainTextToLexical(text) - expect(lexicalToPlainText(body)).toBe(text) - expect(plainTextToLexical(lexicalToPlainText(body))).toEqual(body) + expect(lexicalToMarkdownText(body)).toBe(text) + expect(plainTextToLexical(lexicalToMarkdownText(body))).toEqual(body) expect(lexicalToMarkdown(body, '\n')).toBe(text) }) + + // Readers of the words (reading time, banned-phrase matching) get no markers: + // a bolded half of a phrase must still match the phrase. + it('gives plain text with no markdown to the readers that want words', () => { + const body = plainTextToLexical('- a game **changer** for [you](/x)') + expect(lexicalToPlainText(body)).toBe('a game changer for you') + }) +}) + +describe('inline parsing cost', () => { + // Every unclosed opener used to rescan the rest of the line: quadratic in + // the line's length, reachable from a template save or the writer's output. + it('converts a long line of unclosed emphasis openers in linear time', () => { + for (const unit of ['*a ', '_a ', '**a ', '*a **b ***c ']) { + const line = unit.repeat(Math.ceil(100_000 / unit.length)) + const started = performance.now() + lexicalBodyToHtml(plainTextToLexical(line) as never) + expect(performance.now() - started).toBeLessThan(1000) + } + }) }) describe('payload round trip', () => { From bfd2078e7a3881124af1f0e7fb244d0e8cf1828f Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 00:38:54 -0500 Subject: [PATCH 04/13] fix(ig): exempt evidence-bank-backed first-party claims from the block A first_party_measurement claim that firstPartyOutcome verified against a bank entry the draft cited now carries qualitySource 'evidence-bank', and decidePolicy no longer emits FIRST_PARTY_MEASUREMENT_PRESENT for it. Unbacked first-party claims are still blocked. Co-Authored-By: Claude Opus 5.5 --- cms/src/lib/informationGain/policy.ts | 17 ++++++++-- cms/src/lib/informationGain/types.ts | 7 ++-- docs/information-gain.md | 8 ++--- pipeline/src/informationGain/scorecard.ts | 4 ++- pipeline/test/igPolicy.test.ts | 40 +++++++++++++++++++++++ pipeline/test/igStage.test.ts | 27 +++++++++++++++ 6 files changed, 94 insertions(+), 9 deletions(-) diff --git a/cms/src/lib/informationGain/policy.ts b/cms/src/lib/informationGain/policy.ts index 9998e7bd..e40d7f18 100644 --- a/cms/src/lib/informationGain/policy.ts +++ b/cms/src/lib/informationGain/policy.ts @@ -88,7 +88,7 @@ export const POLICY_FIELDS = [ kind: 'boolean', outcome: 'BLOCK', description: - 'Drafts are written by a model, so any claim that we ran our own test, survey, or study is fabricated. Block it.', + "Drafts are written by a model, so a claim that we ran our own test, survey, or study is fabricated unless it restates an entry in the workspace's evidence bank. Block every other one.", }, { key: 'maxContradictionProbability', @@ -290,6 +290,15 @@ function dedupe(reasons: PolicyReason[]): PolicyReason[] { }) } +/** + * A claim the workspace's evidence bank verified (`firstPartyOutcome` in the + * pipeline). The bank is the one place a draft may get a first-party + * measurement from, so `blockFirstPartyMeasurements` does not apply to it. + */ +const bankBacked = (claim: ClaimRecord): boolean => + claim.verificationMode === 'verified' && + claim.evidence.some((evidence) => evidence.qualitySource === 'evidence-bank') + /** * The gate itself: pure, deterministic, and the only place a decision is made. * Every triggered rule is reported; the decision is the most severe of them. @@ -352,7 +361,11 @@ export function decidePolicy( }) } - if (policy.blockFirstPartyMeasurements && claim.kind === 'first_party_measurement') { + if ( + policy.blockFirstPartyMeasurements && + claim.kind === 'first_party_measurement' && + !bankBacked(claim) + ) { reasons.push({ policy: 'FIRST_PARTY_MEASUREMENT_PRESENT', claimId: claim.id, diff --git a/cms/src/lib/informationGain/types.ts b/cms/src/lib/informationGain/types.ts index b507d2f8..9ca8eb1c 100644 --- a/cms/src/lib/informationGain/types.ts +++ b/cms/src/lib/informationGain/types.ts @@ -117,8 +117,11 @@ export const SOURCE_QUALITY_CLASSES = [ export type SourceQualityClass = (typeof SOURCE_QUALITY_CLASSES)[number] -/** Where a source's quality score came from: the admin table, the rubric, or a capped rubric score. */ -export type QualitySource = 'evidence-sources' | 'rubric' | 'rubric_capped' +/** + * Where a source's quality score came from: the admin table, the rubric, a + * capped rubric score, or the workspace's own evidence bank (`firstPartyOutcome`). + */ +export type QualitySource = 'evidence-sources' | 'rubric' | 'rubric_capped' | 'evidence-bank' export interface Evidence { url: string diff --git a/docs/information-gain.md b/docs/information-gain.md index b46807ee..9fccf63c 100644 --- a/docs/information-gain.md +++ b/docs/information-gain.md @@ -136,7 +136,7 @@ One contradicting excerpt therefore costs a claim twice, the value point and the ## Source quality, and why classifying domains is real work -`resolveSourceQuality` (`sourceQuality.ts`) scores one cited URL two ways, and only two. Either the domain matches an **active row in the `evidence-sources` collection** — an explicit human decision — or it falls back to the judge's rubric guess, which is **capped at `UNKNOWN_DOMAIN_CAP` (0.75)**. Longest domain match wins, so a specific subdomain rule beats its parent. A URL that will not parse scores 0: an unusable citation is not evidence. `first_party_dataset` can never come from the rubric at all; only the table can certify a source as our own. +`resolveSourceQuality` (`sourceQuality.ts`) scores one cited URL two ways, and only two. Either the domain matches an **active row in the `evidence-sources` collection** — an explicit human decision — or it falls back to the judge's rubric guess, which is **capped at `UNKNOWN_DOMAIN_CAP` (0.75)**. Longest domain match wins, so a specific subdomain rule beats its parent. A URL that will not parse scores 0: an unusable citation is not evidence. `first_party_dataset` can never come from the rubric at all; only the table can certify a source as our own. The one other way a claim gets that class is the evidence bank: a claim that restates a bank entry the draft cited is settled by `firstPartyOutcome` without a web search, and its single `Evidence` row carries `qualitySource: 'evidence-bank'`. The class scores (`SOURCE_QUALITY_SCORE`, uncalibrated policy dials): @@ -194,7 +194,7 @@ The eleven thresholds (`POLICY_FIELDS` — the single source of truth from which | `minNumericTemporalIntegrity` | 0.95 | BLOCK | Integrity floor for a materially novel claim containing numbers, dates or units. | | `requireExactValueMatch` | true | BLOCK | A materially novel claim's values must match the evidence exactly (`exactness == 1`). | | `requireEvidenceLineage` | true | BLOCK | A materially novel verifiable claim must cite at least one excerpt. | -| `blockFirstPartyMeasurements` | true | BLOCK | Drafts are model-generated, so a claimed first-party test, survey or dataset is fabricated. | +| `blockFirstPartyMeasurements` | true | BLOCK | Drafts are model-generated, so a claimed first-party test, survey or dataset is fabricated unless the workspace's evidence bank backs it. | | `maxContradictionProbability` | 0.25 | HUMAN_REVIEW | A claim contradicting reliable evidence at or above this needs a human — it may be legitimately new. | | `materialNoveltyThreshold` | 0.55 | HUMAN_REVIEW | Novelty at or above this makes a claim "materially novel". A materially novel *inference* needs review. | | `maxInternalDuplicationRate` | 0.35 | HUMAN_REVIEW | Share of draft claims already published on this site; at or above this, a consolidation review. | @@ -212,7 +212,7 @@ And the eleven codes they produce (`POLICY_CODES`): | `NOVEL_FACTUAL_CLAIM_REQUIRES_SUPPORT` | BLOCK | claim | A verified, materially novel, non-numeric verifiable claim below its integrity floor. | | `NUMERIC_CLAIM_REQUIRES_EXACT_SUPPORT` | BLOCK | claim | Same, for a claim carrying numbers/dates/units — below the numeric floor, or `exactness < 1` under `requireExactValueMatch`. | | `EVIDENCE_LINEAGE_MISSING` | BLOCK | claim | A verified, materially novel verifiable claim with an empty evidence list. | -| `FIRST_PARTY_MEASUREMENT_PRESENT` | BLOCK | claim | The claim's type is `first_party_measurement`. | +| `FIRST_PARTY_MEASUREMENT_PRESENT` | BLOCK | claim | The claim's type is `first_party_measurement` and it is not a `verified` claim with `evidence-bank` evidence. | | `CONTRADICTION_REQUIRES_REVIEW` | HUMAN_REVIEW | claim | `contradictionProbability ≥ maxContradictionProbability`. | | `NOVEL_INFERENCE_REQUIRES_REVIEW` | HUMAN_REVIEW | claim | A materially novel claim of type `inference`. | @@ -329,7 +329,7 @@ Named here so nothing below reads as an oversight: - **Calibration.** Every 0–1 signal is an uncalibrated model estimate and every stored run says so (`calibrated: false`). The thresholds in `POLICY_FIELDS` are defensible starting points, not measurements. Until a labelled set exists, "novelty 0.6" means "this model said 0.6", not "60% of such claims are novel". - **SERP percentiles.** There is no comparison of a draft's scores against the distribution of the pages actually ranking for its keyword. Coverage and gain are absolute numbers against one snapshot, so "good" is a threshold someone chose rather than a position in a field. - **Embeddings.** Duplicate detection is the judge's estimate plus a lexical Jaccard near-duplicate check (`NEAR_DUPLICATE_THRESHOLD` 0.8) inside one draft. Nothing is embedded, so a claim reworded past the lexical threshold relies entirely on the judge noticing. -- **First-party lineage.** `first_party_dataset` is a class the `evidence-sources` table can assign and `blockFirstPartyMeasurements` blocks every claimed first-party measurement outright, because a model-generated draft cannot have run a test. There is no mechanism for a tenant that *does* have first-party data to register a dataset and let a claim cite it. Until there is, the honest answer is the block. +- **First-party lineage.** `first_party_dataset` is a class the `evidence-sources` table can assign, and `blockFirstPartyMeasurements` blocks a claimed first-party measurement because a model-generated draft cannot have run a test. The exception is the evidence bank: a claim that restates a bank entry the draft cited (`firstPartyMatches`, then `firstPartyOutcome`) is verified with `evidence-bank` evidence and is exempt, because the measurement is the workspace's, not the model's. Every other first-party claim is still blocked. - **Excerpt verification for baseline claims.** A baseline claim's `excerpt` is counted, not enforced: unverifiable excerpts are recorded on the snapshot as `pages[].unverifiedExcerptCount` and kept, because dropping them shrinks the baseline and a smaller baseline makes every draft look *more* novel. - **Cross-article verifier caching**, per the Cost section above. - **`robots.txt`, per-host throttling and 429 backoff** in the crawler, per the Fetch limits section above. diff --git a/pipeline/src/informationGain/scorecard.ts b/pipeline/src/informationGain/scorecard.ts index 61870533..4e8135dc 100644 --- a/pipeline/src/informationGain/scorecard.ts +++ b/pipeline/src/informationGain/scorecard.ts @@ -169,7 +169,9 @@ export function firstPartyOutcome(ref: string, excerpt: string): VerificationOut sourceKind: 'first_party_dataset', domain: '', qualityScore: SOURCE_QUALITY_SCORE.first_party_dataset, - qualitySource: 'rubric', + // What exempts the claim from `blockFirstPartyMeasurements`: the bank, + // not a model, is the source of the measurement. + qualitySource: 'evidence-bank', }, ], exactnessMismatches: [], diff --git a/pipeline/test/igPolicy.test.ts b/pipeline/test/igPolicy.test.ts index 8ee0427f..cdf65cd4 100644 --- a/pipeline/test/igPolicy.test.ts +++ b/pipeline/test/igPolicy.test.ts @@ -498,6 +498,46 @@ describe('decidePolicy', () => { assert.deepEqual([...gates.map((g) => g.code)].sort(), [...POLICY_CODES].sort()) }) + // A first-party claim the evidence bank verified is the bank's measurement, + // not the model's, so the block does not apply; every other one still blocks. + describe('first-party measurements and the evidence bank', () => { + const bankEvidence = (): Evidence => ({ + ...evidence(), + url: '', + domain: '', + sourceKind: 'first_party_dataset', + qualityScore: 1, + qualitySource: 'evidence-bank', + }) + const firstParty = (overrides: Partial = {}) => + decidePolicy( + makeScorecard(), + [makeClaim({ id: 'c005', kind: 'first_party_measurement', ...overrides })], + DEFAULT_POLICY, + ) + + it('does not block a claim the evidence bank verified', () => { + const result = firstParty({ evidence: [bankEvidence()] }) + assert.deepEqual(result.reasons, []) + assert.equal(result.decision, 'PASS') + }) + + it('still blocks one backed only by web evidence', () => { + assert.deepEqual( + firstParty().reasons.map((r) => r.policy), + ['FIRST_PARTY_MEASUREMENT_PRESENT'], + ) + }) + + it('still blocks bank evidence on a claim that was not verified', () => { + assert.ok( + firstParty({ evidence: [bankEvidence()], verificationMode: 'not_applicable' }).reasons.some( + (r) => r.policy === 'FIRST_PARTY_MEASUREMENT_PRESENT', + ), + ) + }) + }) + it('stops at BASELINE_UNAVAILABLE even when other gates would fire', () => { const result = decidePolicy( makeScorecard( diff --git a/pipeline/test/igStage.test.ts b/pipeline/test/igStage.test.ts index 717663dc..8638afdc 100644 --- a/pipeline/test/igStage.test.ts +++ b/pipeline/test/igStage.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict' import test from 'node:test' import { + decidePolicy, DEFAULT_POLICY, type ClaimRecord, type DocumentScore, @@ -428,3 +429,29 @@ test('a first-party outcome is verified, first-party sourced, and says it skippe assert.equal(outcome.evidence[0].sourceKind, 'first_party_dataset') assert.match(String(outcome.verifierNotes), /evidence-bank entry E1; not sent to the web verifier/) }) + +test('a first-party claim the evidence bank backs clears the first-party gate', () => { + const claim = draftClaim({ + id: 'c004', + type: 'first_party_measurement', + text: 'Our median onboarding time across 120 customers is 11 days.', + }) + const recordWith = (verification: ReturnType) => + buildClaimRecord( + { claim, judge: deriveJudgeSignals(judgeReply(), cluster), intraDocumentNovelty: 1, verification }, + DEFAULT_POLICY, + ) + const banked = recordWith(firstPartyOutcome('E1', claim.text)) + assert.equal(banked.evidence[0]?.qualitySource, 'evidence-bank') + assert.deepEqual( + decidePolicy(scorecardFor(), [banked], DEFAULT_POLICY).reasons.map((r) => r.policy), + [], + ) + // The same claim without the bank behind it is still a fabricated measurement. + const unbacked = recordWith(unverifiedOutcome('first_party_measurement')) + assert.ok( + decidePolicy(scorecardFor(), [unbacked], DEFAULT_POLICY).reasons.some( + (r) => r.policy === 'FIRST_PARTY_MEASUREMENT_PRESENT', + ), + ) +}) From 8957f8b9dd631c93943b6d5f7adcca83e2cd3a5f Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 00:39:15 -0500 Subject: [PATCH 05/13] fix(ig): clamp suggestClass to secondary at most A verifier rubric guess of primary or official_docs is now suggested as secondary, so accepting a Source review suggestion unchanged can no longer create a floor-clearing rule from a model guess. Co-Authored-By: Claude Opus 5.5 --- cms/src/lib/informationGain/candidates.ts | 17 ++++++++++------- docs/information-gain.md | 2 +- pipeline/test/igCandidates.test.ts | 21 ++++++++++++++++++--- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/cms/src/lib/informationGain/candidates.ts b/cms/src/lib/informationGain/candidates.ts index 8b189684..2d8fde0e 100644 --- a/cms/src/lib/informationGain/candidates.ts +++ b/cms/src/lib/informationGain/candidates.ts @@ -92,12 +92,12 @@ const CANDIDATE_RANK: Record = { * able to produce a class that clears a novel-claim floor by itself: * * - **Cited somewhere** — the verifier's own rubric class, taken modally across - * sightings. That guess is exactly the class the pipeline would have used had - * the unknown-domain cap not applied, so it is the most informative default - * available. `unknown` is ignored, `blocked` is read as `unverified` (a - * suggestion must not default to shutting a domain out), ties resolve to the - * weaker class, and nothing usable falls back to `secondary` — where the cap - * already lands the domain today. + * sightings, then clamped to `secondary` at most. The rubric is a model's + * guess, and `primary` or `official_docs` would clear a floor on one click, so + * the guess can only lower the default. `unknown` is ignored, `blocked` is + * read as `unverified` (a suggestion must not default to shutting a domain + * out), ties resolve to the weaker class, and nothing usable falls back to + * `secondary` — where the cap already lands the domain today. * - **SERP only** — `secondary` when the domain rating clears * `SERP_SECONDARY_MIN_DR`, else `unverified`. A page that ranks is by * definition a secondary treatment of the topic, and domain rating measures @@ -117,7 +117,10 @@ export function suggestClass(sightings: CandidateSighting[]): CandidateClass { }) .filter((kind): kind is CandidateClass => kind !== null) - if (rubric.length > 0) return modalBy(rubric, (value) => CANDIDATE_RANK[value]) ?? 'secondary' + if (rubric.length > 0) { + const modal = modalBy(rubric, (value) => CANDIDATE_RANK[value]) ?? 'secondary' + return CANDIDATE_RANK[modal] > CANDIDATE_RANK.secondary ? 'secondary' : modal + } if (sightings.some((sighting) => sighting.kind === 'cited')) return 'secondary' const rating = sightings.reduce( diff --git a/docs/information-gain.md b/docs/information-gain.md index 9fccf63c..6f0faf30 100644 --- a/docs/information-gain.md +++ b/docs/information-gain.md @@ -174,7 +174,7 @@ Two things become candidates, both filtered through `matchEvidenceRule` so a rat `collectCandidateSightings` (`cms/src/lib/informationGain/candidates.ts`) builds one sighting per domain per kind, `recordCandidateSightings` (`pipeline/src/informationGain/candidates.ts`) upserts one row per domain, and both run after the run row is created so each sighting carries a real `runId`. The write cannot change a decision — it happens after `decidePolicy` — and it cannot cost one either: a failure is caught and reported through `StageOutcome.warnings`, which `runPipeline` logs, counts as `warned` beside `failed`, and stores on the article's audit row. Throwing instead would strand the article at `qa_passed` and re-buy all three LLM passes on the next run over a queue row. -**The suggestion is a default, not a verdict.** `suggestClass` uses the verifier's own rubric class where there is one — modally across sightings, `unknown` ignored, `blocked` read as `unverified`, ties resolving to the weaker class — and falls back to `secondary` above `SERP_SECONDARY_MIN_DR` (40) or `unverified` below it for a domain only ever seen ranking. It can never return `first_party_dataset` (only a human certifies a source as ours) or `blocked`, and both classes it can return sit at or below `UNKNOWN_DOMAIN_CAP` — so accepting a suggestion unchanged records a judgement without clearing any floor. Only a deliberate choice of `primary`, `official_docs` or `first_party_dataset` does that. +**The suggestion is a default, not a verdict.** `suggestClass` uses the verifier's own rubric class where there is one — modally across sightings, `unknown` ignored, `blocked` read as `unverified`, ties resolving to the weaker class, then clamped to `secondary` at most, so a rubric guess of `primary` or `official_docs` is suggested as `secondary` — and falls back to `secondary` above `SERP_SECONDARY_MIN_DR` (40) or `unverified` below it for a domain only ever seen ranking. It can never return `first_party_dataset` (only a human certifies a source as ours) or `blocked`, and both classes it can return sit at or below `UNKNOWN_DOMAIN_CAP` — so accepting a suggestion unchanged records a judgement without clearing any floor. Only a deliberate choice of `primary`, `official_docs` or `first_party_dataset` does that. **`evidence-sources` stays the authority on what is rated.** A candidate's `status` is never consulted for that: the page re-checks every pending row against the live rules on each render, so a rule added by hand takes its domain off the queue and deactivating one puts it back, with no write on either side. The pipeline applies the same rule from the other end — a sighting on an `approved` row means its rule is no longer active, so the row returns to `pending`. A dismissal is a standing answer and is never reopened automatically, though counts and `lastSeenAt` keep moving so a domain that keeps recurring stays visible. diff --git a/pipeline/test/igCandidates.test.ts b/pipeline/test/igCandidates.test.ts index 0a67934e..5cd67e55 100644 --- a/pipeline/test/igCandidates.test.ts +++ b/pipeline/test/igCandidates.test.ts @@ -162,7 +162,22 @@ describe('collectCandidateSightings', () => { describe('suggestClass', () => { it('uses the verifier rubric a cited domain was given', () => { - assert.equal(suggestClass([sighting({ sourceKind: 'official_docs' })]), 'official_docs') + assert.equal(suggestClass([sighting({ sourceKind: 'unverified' })]), 'unverified') + assert.equal(suggestClass([sighting({ sourceKind: 'secondary' })]), 'secondary') + }) + + // A model's guess must never clear a floor on one click. + it('clamps a floor-clearing rubric guess to secondary', () => { + assert.equal(suggestClass([sighting({ sourceKind: 'official_docs' })]), 'secondary') + assert.equal(suggestClass([sighting({ sourceKind: 'primary' })]), 'secondary') + assert.equal( + suggestClass([ + sighting({ sourceKind: 'primary', runId: 1 }), + sighting({ sourceKind: 'primary', runId: 2 }), + sighting({ sourceKind: 'unverified', runId: 3 }), + ]), + 'secondary', + ) }) it('takes the most common rubric across sightings', () => { @@ -205,8 +220,8 @@ describe('suggestClass', () => { // Being cited says more than ranking does, so a rubric guess wins outright. it('prefers a citation rubric over the SERP rating', () => { assert.equal( - suggestClass([sighting({ sourceKind: 'primary' }), sighting({ kind: 'serp', domainRating: 78 })]), - 'primary', + suggestClass([sighting({ sourceKind: 'unverified' }), sighting({ kind: 'serp', domainRating: 78 })]), + 'unverified', ) }) }) From e5bc5af06568ef25f9a68e3ad8c8c159dab4a59f Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 00:40:16 -0500 Subject: [PATCH 06/13] fix(tenant): read [F]/[R] as evidence refs only when the bank has them evidenceRefsIn and stripEvidenceRefs take an optional { bank }. With it, [E] is always a marker (a hallucinated [E9] still reaches QA as unknown), while [F] and [R] are markers only when that ref exists in the bank; otherwise ordinary prose like "[F5]" is left untouched. Without the option the behaviour is unchanged until generate.ts passes the bank. Co-Authored-By: Claude Opus 5.5 --- cms/src/lib/tenant/evidenceBank.ts | 47 +++++++++++++++++++++++++++--- pipeline/test/evidenceBank.test.ts | 33 +++++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/cms/src/lib/tenant/evidenceBank.ts b/cms/src/lib/tenant/evidenceBank.ts index 69375223..3f8c8505 100644 --- a/cms/src/lib/tenant/evidenceBank.ts +++ b/cms/src/lib/tenant/evidenceBank.ts @@ -422,13 +422,48 @@ export function evidenceBankSummary( */ const REF_PATTERN = /\[([EFR]\d+)\]/gi +/** A marker with the spaces or tabs that carry it, for stripping. */ +const REF_WITH_SPACE_PATTERN = /[ \t]*\[([EFR]\d+)\]/gi + +/** + * Which bank the markers in a piece of text are read against. Omitted, every + * `[E]`, `[F]` and `[R]` is a marker, as before the bank was taken + * into account; callers that have the stage's bank should pass it. + */ +export interface EvidenceRefOptions { + bank: EvidenceBankContent | null | undefined +} + +/** + * Whether one bracketed ref is a citation marker rather than prose. + * + * `[E]` always is, so a hallucinated `[E9]` still reaches QA as an unknown + * ref. `[F]` and `[R]` also read naturally as ordinary text (a figure, a + * function key, a footnote), so with a bank given they are markers only when + * that ref exists in it; otherwise they are left in the text untouched. + */ +function isEvidenceMarker(ref: string, opts: EvidenceRefOptions | undefined): boolean { + if (opts === undefined || ref.startsWith('E')) return true + const bank = opts.bank + if (!bank) return false + return ( + bank.verifiedClaims.some((row) => row.ref === ref) || + bank.facts.some((row) => row.ref === ref) || + bank.rejectedClaims.some((row) => row.ref === ref) + ) +} + /** Every evidence ref cited in a piece of generated text, de-duplicated in order. */ -export function evidenceRefsIn(text: string | null | undefined): string[] { +export function evidenceRefsIn( + text: string | null | undefined, + opts?: EvidenceRefOptions, +): string[] { if (!text) return [] const seen = new Set() const refs: string[] = [] for (const match of text.matchAll(REF_PATTERN)) { const ref = normaliseEvidenceRef(match[1]) + if (!isEvidenceMarker(ref, opts)) continue if (seen.has(ref)) continue seen.add(ref) refs.push(ref) @@ -446,10 +481,14 @@ export function evidenceRefsIn(text: string | null | undefined): string[] { * protocol between the prompt and QA, and no reader should ever see one, so * this runs on every generated string field before anything is stored — in * either case, since a model that types `[e3]` must not leave the marker in the - * published text just because it shouted less than the prompt did. + * published text just because it shouted less than the prompt did. With a bank + * given, an `[F]` or `[R]` the bank does not have is prose and stays (see + * `isEvidenceMarker`). */ -export function stripEvidenceRefs(text: string): string { - return text.replace(/[ \t]*\[([EFR]\d+)\]/gi, '') +export function stripEvidenceRefs(text: string, opts?: EvidenceRefOptions): string { + return text.replace(REF_WITH_SPACE_PATTERN, (marker, raw: string) => + isEvidenceMarker(normaliseEvidenceRef(raw), opts) ? '' : marker, + ) } /** What the deterministic half of the evidence check found about one ref. */ diff --git a/pipeline/test/evidenceBank.test.ts b/pipeline/test/evidenceBank.test.ts index ae92da81..67e8fdef 100644 --- a/pipeline/test/evidenceBank.test.ts +++ b/pipeline/test/evidenceBank.test.ts @@ -297,6 +297,39 @@ test('stripEvidenceRefs removes the marker and its leading space, and nothing el assert.equal(stripEvidenceRefs('## Speed [E1]'), '## Speed') }) +test('with a bank, [F] and [R] are markers only when the bank has that ref', () => { + const withRefs = { + bank: bank({ + verifiedClaims: [claim({ ref: 'E1' })], + facts: [{ ref: 'F2', fact: 'Founded in 2021', source: '', owner: '', lastConfirmedAt: '' }], + rejectedClaims: [ + { ref: 'R3', claim: 'The fastest', status: 'rejected', reason: '', replacement: '' }, + ], + }), + } + const prose = 'Press [F5] to reload, see figure [F12] and region [R9]; we hold [E1] [F2] [r3].' + assert.deepEqual(evidenceRefsIn(prose, withRefs), ['E1', 'F2', 'R3']) + assert.equal( + stripEvidenceRefs(prose, withRefs), + 'Press [F5] to reload, see figure [F12] and region [R9]; we hold.', + ) + // No bank at all: no F or R ref can exist, so none is a marker. + assert.deepEqual(evidenceRefsIn('Press [F5] and [R1].', { bank: null }), []) + assert.equal(stripEvidenceRefs('Press [F5] and [R1].', { bank: null }), 'Press [F5] and [R1].') +}) + +test('with a bank, [E] is always a marker, so a hallucinated one still reaches QA', () => { + const opts = { bank: bank({ verifiedClaims: [claim({ ref: 'E1' })] }) } + const refs = evidenceRefsIn('It holds [E1] and [e9].', opts) + assert.deepEqual(refs, ['E1', 'E9']) + assert.equal(stripEvidenceRefs('It holds [E1] and [e9].', opts), 'It holds and.') + assert.deepEqual( + checkEvidenceRefs(refs, opts.bank, { asOf: '2026-09-02', surface: 'web' }).unknown, + ['E9'], + ) + assert.deepEqual(evidenceRefsIn('It holds [E9].', { bank: null }), ['E9']) +}) + test('checkEvidenceRefs separates usable, unknown, and unusable refs', () => { const checked = bank({ verifiedClaims: [ From e0bca8a3b8d8e65c9983d84ee7e31de875883a2c Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 00:41:05 -0500 Subject: [PATCH 07/13] fix(ig): refuse a judge entry with no numeric internalDuplicateProbability The judge prompt requests the field in every call, so a missing or non-numeric value is refused like duplicateProbability instead of being read as 0, which hid internal duplicates from review. Co-Authored-By: Claude Opus 5.5 --- cms/src/lib/informationGain/parsers.ts | 14 +++++++----- pipeline/test/igParsers.test.ts | 30 +++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/cms/src/lib/informationGain/parsers.ts b/cms/src/lib/informationGain/parsers.ts index bb0e45f4..02f9c853 100644 --- a/cms/src/lib/informationGain/parsers.ts +++ b/cms/src/lib/informationGain/parsers.ts @@ -376,11 +376,15 @@ export function parseJudgeReply( // Novelty is `1 - duplicateProbability`, so reading a missing or quoted // value as 0 would call the claim maximally novel and send it to the paid // verifier. Refuse the reply instead; the stage retries on the next run. - if ( - typeof record.duplicateProbability !== 'number' || - !Number.isFinite(record.duplicateProbability) - ) { - throw new Error(`judge reply for claim ${claimId} has no numeric duplicateProbability`) + // The internal score is asked for in every judge call, internal corpus or + // not, and a missing one read as 0 would hide a duplicate of our own + // published article from `INTERNAL_DUPLICATION_REQUIRES_REVIEW`, so it is + // refused the same way. + for (const field of ['duplicateProbability', 'internalDuplicateProbability'] as const) { + const value = record[field] + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`judge reply for claim ${claimId} has no numeric ${field}`) + } } const utility = asRecord(record.utility) ?? {} signals.set(claimId, { diff --git a/pipeline/test/igParsers.test.ts b/pipeline/test/igParsers.test.ts index dc872c4c..3bfe23a0 100644 --- a/pipeline/test/igParsers.test.ts +++ b/pipeline/test/igParsers.test.ts @@ -488,6 +488,7 @@ describe('parseJudgeReply', () => { { claimId: 'c001', duplicateProbability: 0, + internalDuplicateProbability: 0, closestBaselineClaimId: 'not-a-known-claim', closestInternalClaimId: 42, containsNumericOrTemporalClaim: 'true', @@ -522,7 +523,7 @@ describe('parseJudgeReply', () => { assert.throws( () => parseJudgeReply( - { claims: [{ claimId: 'c001', duplicateProbability: 0.2 }] }, + { claims: [{ claimId: 'c001', duplicateProbability: 0.2, internalDuplicateProbability: 0 }] }, ['c001', 'c002'], queryIds, baselineIds, @@ -531,7 +532,12 @@ describe('parseJudgeReply', () => { ) const signals = parseJudgeReply( - { claims: [{ claimId: 'c001', duplicateProbability: 0.2 }, { claimId: 'c999' }] }, + { + claims: [ + { claimId: 'c001', duplicateProbability: 0.2, internalDuplicateProbability: 0 }, + { claimId: 'c999' }, + ], + }, ['c001'], queryIds, baselineIds, @@ -546,7 +552,7 @@ describe('parseJudgeReply', () => { assert.throws( () => parseJudgeReply( - { claims: [{ claimId: 'c001', duplicateProbability }] }, + { claims: [{ claimId: 'c001', duplicateProbability, internalDuplicateProbability: 0 }] }, ['c001'], queryIds, baselineIds, @@ -555,6 +561,24 @@ describe('parseJudgeReply', () => { ) } }) + + // The judge prompt asks for it in every call. Read as 0, a missing value + // would hide a duplicate of our own published article from the + // internal-duplication review. + it('refuses an entry whose internalDuplicateProbability is missing or not a number', () => { + for (const internalDuplicateProbability of [undefined, '0.9', null, Number.NaN]) { + assert.throws( + () => + parseJudgeReply( + { claims: [{ claimId: 'c001', duplicateProbability: 0.2, internalDuplicateProbability }] }, + ['c001'], + queryIds, + baselineIds, + ), + { message: 'judge reply for claim c001 has no numeric internalDuplicateProbability' }, + ) + } + }) }) describe('parseVerifierReply', () => { From 17a5fc99b775789a33683e1dfe5d751ca615830e Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 00:52:03 -0500 Subject: [PATCH 08/13] fix(ig): read range bounds with the range kind and compound word numbers as one amount 5-10% now reads as two percents and $5-10 as two currency amounts, so they match 5% to 10% and $5 to $10. The re-kinded bound is flagged alsoBareNumber and still matches whatever the bare number matched, and anchors are chosen with the bare reading, so matching only gains. Tokenisation is unchanged; only two shapes are relabelled, and years, signed numbers, figures glued to letters, suffixed amounts, to-ranges and spaced em dashes read as before. twenty-five and twenty five read as 25 rather than 5 (or as a 20-5 range). Co-Authored-By: Claude Opus 5.5 --- cms/src/lib/informationGain/exactness.ts | 166 +++++++++++++++++++++-- docs/information-gain.md | 2 + pipeline/test/igExactness.test.ts | 107 +++++++++++++++ 3 files changed, 264 insertions(+), 11 deletions(-) diff --git a/cms/src/lib/informationGain/exactness.ts b/cms/src/lib/informationGain/exactness.ts index c6be87f5..2444040c 100644 --- a/cms/src/lib/informationGain/exactness.ts +++ b/cms/src/lib/informationGain/exactness.ts @@ -21,6 +21,13 @@ export interface ExtractedValue { unit: string | null /** The text the value was read from, quoted back in mismatch messages. */ raw: string + /** + * Set on a range bound that used to be read as a bare number (`5` in + * `5-10%`, `10` in `$5-10`) and now carries the range's kind. It still + * matches whatever that bare number matched, so the new kind only adds + * matches (see `rangeBoundKinds` and `sameValue`). + */ + alsoBareNumber?: true } export interface TextValues { @@ -30,7 +37,11 @@ export interface TextValues { comparative: 'more' | 'less' | 'equal' | null } -/** Word numbers we accept. Deliberately small: one…twenty plus the round tens. */ +/** + * Word numbers we accept. Deliberately small: one…twenty plus the round tens, + * and a round ten joined to a unit (`twenty-five`, `twenty five`), which reads + * as one amount rather than as its last word. + */ const WORD_NUMBERS: Record = { one: 1, two: 2, @@ -157,7 +168,19 @@ const byLengthDesc = (tokens: string[]): string => const MONTH_SRC = byLengthDesc(Object.keys(MONTHS)) const UNIT_SRC = byLengthDesc(Object.keys(UNIT_CANONICAL)) -const WORD_NUMBER_SRC = byLengthDesc(Object.keys(WORD_NUMBERS)) +const TENS = ['twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] +const TENS_SRC = TENS.join('|') +const UNITS_SRC = 'one|two|three|four|five|six|seven|eight|nine' +/** Every other word number. The round tens are left out: they take the guarded form below. */ +const WORD_NUMBER_SRC = byLengthDesc(Object.keys(WORD_NUMBERS).filter((word) => !TENS.includes(word))) +const COMPOUND_JOIN = String.raw`(?:-|\s)` +/** + * `twenty-five` or `twenty five`. A bare round ten may not be followed by a + * unit word: without that lookahead the regex backtracks to `twenty` alone and + * reads `twenty-five minutes` as the range 20 to 5 minutes. + */ +const WORD_AMOUNT_SRC = String.raw`(?:(?:${TENS_SRC})${COMPOUND_JOIN}(?:${UNITS_SRC})|(?:${TENS_SRC})(?!${COMPOUND_JOIN}(?:${UNITS_SRC})\b)|${WORD_NUMBER_SRC})` +const COMPOUND_PATTERN = new RegExp(String.raw`^(${TENS_SRC})${COMPOUND_JOIN}(${UNITS_SRC})$`) /** `1.5`, `20,000`, `20K`, `1.5 million`. A bare `k`/`m`/`bn` must touch the digits. */ const DIGIT_AMOUNT = String.raw`\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?` @@ -190,7 +213,7 @@ const NUMERIC_AMOUNT = String.raw`(?:${SIGN})?(?:${DIGIT_AMOUNT})(?:${MULTIPLIER * token as a measurement. A bare "one of the best ways" is prose, and reading a * value out of it would manufacture exactness mismatches from ordinary text. */ -const AMOUNT = String.raw`(?:${NUMERIC_AMOUNT}|\b(?:${WORD_NUMBER_SRC})\b)` +const AMOUNT = String.raw`(?:${NUMERIC_AMOUNT}|\b${WORD_AMOUNT_SRC}\b)` /** * One left-to-right scan; alternatives are tried in order at each position, so @@ -245,6 +268,8 @@ function parseAmount(raw: string | undefined): number | null { // `+` is emphasis; only a minus (ASCII or U+2212) flips the value. return digits[1] === '-' || digits[1] === '−' ? -scaled : scaled } + const compound = COMPOUND_PATTERN.exec(text) + if (compound) return WORD_NUMBERS[compound[1]] + WORD_NUMBERS[compound[2]] return WORD_NUMBERS[text] ?? null } @@ -342,27 +367,114 @@ function comparativeOf(text: string): TextValues['comparative'] { return match[2] !== undefined ? 'less' : 'equal' } +/** One scanned match: what it produced, which rule produced it, and where. */ +interface Token { + values: ExtractedValue[] + rule: 'bare' | 'pct' | 'currencyPrefix' | 'other' + start: number + end: number +} + +function ruleOf(match: RegExpExecArray): Token['rule'] { + const g = match.groups ?? {} + if (g.bare !== undefined) return 'bare' + if (g.pct !== undefined) return 'pct' + if (g.symbol !== undefined || g.codePre !== undefined) return 'currencyPrefix' + return 'other' +} + +/** What joins the two bounds of `5-10%` or `$5-10`. A spaced em dash is prose, not a range. */ +const RANGE_JOIN = /^(?:\s*[-–]\s*|—)$/ + +/** An unsigned figure in digits, the only shape a re-kinded bound may have. */ +const UNSIGNED_DIGITS = /^\d/ + +/** A bare number, as the scan reads one: no unit, not a year. */ +const isBareNumber = (token: Token): boolean => + token.rule === 'bare' && + token.values.length === 1 && + token.values[0].kind === 'number' && + UNSIGNED_DIGITS.test(token.values[0].raw) + +/** + * Gives the bare bound of a two-sided range the kind its other bound states. + * + * The scan reads `5-10%` as the number 5 and the percent 10, and `$5-10` as $5 + * and the number 10, so `5-10%` only half matches `5% to 10%`. Re-kinding the + * bare bound outright broke pairings that match today (`5-10%` against + * `between 5 and 10%`, `$10-20 seats` against `$10 for 20 seats`), so the bound + * keeps its bare reading as well: `alsoBareNumber` makes `sameValue` accept + * whatever the bare number accepted. Matching only ever gains. + * + * Tokenisation is untouched; this only relabels a value the scan already + * produced, and only in two shapes, joined by `-` or `–` (spaces allowed) or an + * unspaced `—`: + * + * - **`5-10%`** — a bare unsigned number directly before an unsigned percent. + * Not after a letter, digit, `.`, `,`, sign or dash (`Q3-5%`, `FY24-25%`, + * `v2-3%`), and never a year (`In 2023 - 15%`, `2024-25%` stay as they were), + * which the scan already reads as a year rather than a bare number. + * - **`$5-10`** — a bare unsigned number directly after a symbol- or + * code-prefixed amount (`$5`, `USD 5`). A bound the scan read with a suffix + * (`$5-10 EUR`, `$5-10k`, `$5-10%`, `$5-10 kg`) or as a year (`$5-2024`) is + * not bare and is left alone, and so is one glued to a letter (`$5-10x`) or + * after a scaled amount (`$5bn-10`, where the 10 is probably billions too). + */ +function rangeBoundKinds(tokens: Token[], text: string): void { + for (let i = 0; i + 1 < tokens.length; i += 1) { + const lo = tokens[i] + const hi = tokens[i + 1] + if (!RANGE_JOIN.test(text.slice(lo.end, hi.start))) continue + + if ( + isBareNumber(lo) && + hi.rule === 'pct' && + UNSIGNED_DIGITS.test(hi.values[0]?.raw ?? '') && + !/[\w.,+\-−–—]/.test(text.slice(lo.start - 1, lo.start)) + ) { + lo.values[0] = { ...lo.values[0], kind: 'percent', unit: '%', alsoBareNumber: true } + } else if ( + lo.rule === 'currencyPrefix' && + lo.values.length === 1 && + isBareNumber(hi) && + !/(?:k|m|bn|thousand|million|billion)$/i.test(lo.values[0].raw) && + !/^[a-z]/i.test(text.slice(hi.end)) + ) { + const unit = lo.values[0].unit + hi.values[0] = { ...hi.values[0], kind: 'currency', unit, alsoBareNumber: true } + } + } +} + /** Pulls every comparable signal out of one excerpt. Pure and deterministic. */ export function extractValues(text: string): TextValues { const source = typeof text === 'string' ? text : '' // Curly apostrophes would otherwise hide "can’t" from the negation scan. const normalised = source.replace(/[‘’]/g, "'") - const values: ExtractedValue[] = [] + const tokens: Token[] = [] VALUE_PATTERN.lastIndex = 0 let match = VALUE_PATTERN.exec(normalised) while (match !== null) { const produced = valueFromMatch(match) - if (Array.isArray(produced)) values.push(...produced) - else if (produced !== null) values.push(produced) + const values = Array.isArray(produced) ? produced : produced === null ? [] : [produced] + if (values.length > 0) { + tokens.push({ + values, + rule: ruleOf(match), + start: match.index, + end: match.index + match[0].length, + }) + } // Zero-length matches cannot happen with these alternatives, but a stuck // lastIndex would loop forever, so step past it defensively. if (match[0].length === 0) VALUE_PATTERN.lastIndex += 1 match = VALUE_PATTERN.exec(normalised) } + rangeBoundKinds(tokens, normalised) return { - values, + values: tokens.flatMap((token) => token.values), negated: NEGATION_PATTERN.test(normalised), direction: directionOf(normalised), comparative: comparativeOf(normalised), @@ -375,6 +487,24 @@ export function extractValues(text: string): TextValues { * so evidence reading `growth was 10%` cannot support `growth was -10%`. */ function sameValue(a: ExtractedValue, b: ExtractedValue): boolean { + if (sameReading(a, b)) return true + // A re-kinded range bound also matches as the bare number it used to be read + // as, so the range kind can only add matches (see `rangeBoundKinds`). + const a2 = a.alsoBareNumber ? asBareNumber(a) : a + const b2 = b.alsoBareNumber ? asBareNumber(b) : b + if (a2 === a && b2 === b) return false + return sameReading(a2, b) || sameReading(a, b2) || sameReading(a2, b2) +} + +/** The value a re-kinded range bound was read as before it took the range's kind. */ +const asBareNumber = (v: ExtractedValue): ExtractedValue => ({ + kind: 'number', + value: v.value, + unit: null, + raw: v.raw, +}) + +function sameReading(a: ExtractedValue, b: ExtractedValue): boolean { // `2000 participants` is classified as a year and `2,000 participants` as a // number; they are the same figure, so the two kinds compare by value. Every // other pair of kinds stays distinct. @@ -449,6 +579,18 @@ function qualifiersCompatible(claim: TextValues, e: TextValues): boolean { const attests = (e: TextValues, value: ExtractedValue): boolean => e.values.some((candidate) => sameValue(value, candidate)) +/** + * `attests` with every re-kinded range bound read as the bare number it used to + * be. Anchors are chosen this way, so a range kind only ever adds value matches + * and never changes which excerpts answer the qualifier checks: an excerpt that + * carries `$10` does not start contradicting the direction of `fell $5-10` + * just because the bound now reads as currency. + */ +const attestsAsBare = (e: TextValues, value: ExtractedValue): boolean => + e.values.some((candidate) => sameReading(bareReading(value), bareReading(candidate))) + +const bareReading = (v: ExtractedValue): ExtractedValue => (v.alsoBareNumber ? asBareNumber(v) : v) + /** Said in place of a qualifier verdict when no excerpt carries the claim's values. */ const UNANCHORED = "no evidence excerpt carries the claim's values" @@ -470,9 +612,11 @@ const UNANCHORED = "no evidence excerpt carries the claim's values" * compatible with every qualifier the claim states. An excerpt that quotes the * figure while contradicting a qualifier does not support it, and gets its own * mismatch message saying which of the two it failed on. - * - **anchors** — the excerpts that carry at least one of the claim's values. - * The three qualifier checks are answered only by anchors, so an excerpt that - * never mentions the figure can no longer vouch for its polarity. + * - **anchors** — the excerpts that carry at least one of the claim's values, + * reading a re-kinded range bound as the bare number it used to be + * (`attestsAsBare`). The three qualifier checks are answered only by anchors, + * so an excerpt that never mentions the figure can no longer vouch for its + * polarity. * - **direction** — one comparable when the claim states one; matched unless the * anchors state the opposite direction and never the claim's. * - **negation** — compared *symmetrically*: the check runs whenever the claim @@ -505,7 +649,7 @@ export function compareValues( const anchors = claim.values.length === 0 ? evidence - : evidence.filter((e) => claim.values.some((value) => attests(e, value))) + : evidence.filter((e) => claim.values.some((value) => attestsAsBare(e, value))) const anchored = claim.values.length === 0 || anchors.length > 0 for (const value of claim.values) { diff --git a/docs/information-gain.md b/docs/information-gain.md index 6f0faf30..2c6e6c65 100644 --- a/docs/information-gain.md +++ b/docs/information-gain.md @@ -130,6 +130,8 @@ The rules that surprise people, all deliberate: - **`more than` / `less than` are thresholds**: a bare `10%` does not support "more than 10%". `equal` is what a bare figure already asserts, so an excerpt with no comparative of its own supports it. - A claim with values and no anchoring excerpt scores 0 — which is why an empty evidence list scores 0 for any claim carrying a number. - Sign handling: `-`, `−` and `+` count as signs only at the start of a token, so `5-10 seconds` is two positive bounds and `top-10 list` is 10, not −10. `+10` parses as 10. +- **A bare range bound takes the range's kind.** `5-10%` reads as two percents and `$5-10` as two dollar amounts, so they match `5% to 10%` and `$5 to $10`. Only two shapes qualify, joined by `-` or `–` (spaces allowed) or an unspaced `—`: an unsigned bare number before an unsigned percent, and an unsigned bare number after a `$`/`€`/`£` or `USD`/`EUR`/`GBP`-prefixed amount. Years (`In 2023 - 15%`), signed numbers (`+2024-25%`), figures glued to a letter (`Q3-5%`, `FY24-25%`, `$5-10x`), amounts with their own suffix (`$5-10 EUR`, `$5-10k`), `to`-ranges, and a spaced em dash (`$10 — 20 seats`) read as they always did. A re-kinded bound is flagged `alsoBareNumber` and still matches whatever the bare number matched (`5-10%` against `between 5 and 10%`, `$10-20 seats` against `$10 for 20 seats`), and anchors are chosen with the bare reading, so the change can only add matches. +- **Word numbers** count only next to a unit, percent, currency or range. A round ten joined to a unit, `twenty-five` or `twenty five`, is one amount (25), never a range of 20 to 5 or just its last word. - Dates compare at month resolution: `2026-01-05` and `Jan 2026` are both 202601. One contradicting excerpt therefore costs a claim twice, the value point and the qualifier point. That is intentional: the gate only asks whether exactness is `< 1`, and the score itself is an uncalibrated signal, not a calibrated probability. diff --git a/pipeline/test/igExactness.test.ts b/pipeline/test/igExactness.test.ts index db43272e..631f64ae 100644 --- a/pipeline/test/igExactness.test.ts +++ b/pipeline/test/igExactness.test.ts @@ -535,3 +535,110 @@ describe('compareValues — one excerpt must carry both value and polarity', () assert.equal(exactness('this is not recommended', ['this is not recommended']), 1) }) }) + +describe('extractValues — compound word numbers', () => { + it('reads a round ten joined to a unit as one amount', () => { + assert.deepEqual(shape('twenty-five percent'), ['percent:25 %']) + assert.deepEqual(shape('twenty five percent'), ['percent:25 %']) + assert.deepEqual(shape('ninety-nine percent'), ['percent:99 %']) + assert.deepEqual(shape('$twenty-five'), ['currency:25 USD']) + assert.deepEqual(shape('fifty-five percent'), ['percent:55 %']) + }) + + // Without the guard the regex backtracks to `twenty` and reads a range. + it('does not split a compound into a range', () => { + assert.deepEqual(shape('twenty-five minutes'), ['number:25 min']) + assert.deepEqual(shape('sixty-one kg'), ['number:61 kg']) + assert.deepEqual(shape('thirty-five to forty minutes'), ['number:35 min', 'number:40 min']) + }) + + it('still needs a measurement around it, and adds no value the scan did not read before', () => { + assert.deepEqual(shape('twenty-five'), []) + assert.deepEqual(shape('ninety-nine'), []) + assert.deepEqual(shape('thirty-five to forty percent'), ['percent:40 %']) + assert.deepEqual(shape('thirty—five to forty percent'), ['percent:40 %']) + assert.deepEqual(shape('thirty−five to forty percent'), ['percent:40 %']) + }) + + it('compares as the amount it states', () => { + assert.equal(exactness('twenty-five percent', ['25% of users']), 1) + assert.equal(exactness('twenty-five percent', ['20% of users']), 0) + assert.equal(exactness('fifty-five percent', ['50%']), 0) + assert.equal(exactness('$thirty-five to $forty', ['$35 and $40']), 1) + }) +}) + +describe('extractValues — range bounds take the range kind', () => { + it('gives the bare lower bound of a percent range the percent kind', () => { + for (const text of ['5-10%', '5 - 10%', '5–10%', '5—10%', 'up 5-10 per cent', '2024.5-2025.5%']) { + assert.equal(shape(text)[0]?.startsWith('percent:'), true, text) + } + const [lo] = extractValues('costs fell 5-10%').values + assert.equal(lo?.alsoBareNumber, true) + }) + + it('gives the bare upper bound of a currency range the currency kind', () => { + assert.deepEqual(shape('$5-10'), ['currency:5 USD', 'currency:10 USD']) + assert.deepEqual(shape('Bags cost $5-10.'), ['currency:5 USD', 'currency:10 USD']) + assert.deepEqual(shape('costs $5-10, depending on size'), ['currency:5 USD', 'currency:10 USD']) + assert.deepEqual(shape('$5-10.50'), ['currency:5 USD', 'currency:10.5 USD']) + assert.deepEqual(shape('$5-10,000'), ['currency:5 USD', 'currency:10000 USD']) + assert.deepEqual(shape('USD 5-10'), ['currency:5 USD', 'currency:10 USD']) + }) + + // Each of these was broken by an earlier attempt; each still reads exactly + // as it did before the range kind existed. + it('leaves every shape that is not a two-sided bare range alone', () => { + const unchanged: [string, string[]][] = [ + ['By 2030 — 40% of companies', ['year:2030', 'percent:40 %']], + ['In 2023 - 15% of firms', ['year:2023', 'percent:15 %']], + ['revenue in 2024-25% growth', ['year:2024', 'percent:25 %']], + ['Q3 2024-25% growth', ['number:3', 'year:2024', 'percent:25 %']], + ['+2024-25%', ['number:2024', 'percent:25 %']], + ['-2024-25%', ['number:-2024', 'percent:25 %']], + ['+2024 to 25%', ['number:2024', 'percent:25 %']], + ['Prices start at $10 — 20 seats included', ['currency:10 USD', 'number:20']], + ['$5-10k seconds', ['currency:5 USD', 'number:10000']], + ['$5-10 EUR', ['currency:5 USD', 'currency:10 EUR']], + ['$5 to 10 people', ['currency:5 USD', 'number:10']], + ['grew from 5 million to 10% share', ['number:5000000', 'percent:10 %']], + ['grew from one to 15% of revenue', ['percent:15 %']], + ['COVID-19 to 20% of', ['number:19', 'percent:20 %']], + ['Q3-5%', ['number:3', 'percent:5 %']], + ['FY24-25% growth', ['number:24', 'percent:25 %']], + ['v2-3%', ['number:2', 'percent:3 %']], + ['from 5 to 10 to 15%', ['number:5', 'number:10', 'percent:15 %']], + ['between 5 and 10%', ['number:5', 'percent:10 %']], + ['20%-30%', ['percent:20 %', 'percent:-30 %']], + ['$5-2024', ['currency:5 USD', 'year:2024']], + ['$1500-2000', ['currency:1500 USD', 'year:2000']], + ['5 -10%', ['number:5', 'percent:-10 %']], + ['5 — 10%', ['number:5', 'percent:10 %']], + ['$5bn-10', ['currency:5000000000 USD', 'number:10']], + ['costs $5-10x more', ['currency:5 USD', 'number:10']], + ] + for (const [text, expected] of unchanged) assert.deepEqual(shape(text), expected, text) + }) + + it('matches the same-kind form and everything the bare number matched', () => { + assert.equal(exactness('costs fell 5-10%', ['costs fell 5% to 10%']), 1) + assert.equal(exactness('costs fell 5-10%', ['costs fell between 5 and 10%']), 1) + assert.equal(exactness('costs fell 5-10%', ['costs fell 5 - 10%']), 1) + assert.equal(exactness('costs fell between 5 and 10%', ['costs fell 5-10%']), 1) + assert.equal(exactness('Bags cost $5-10', ['bags cost $5 to $10']), 1) + assert.equal(exactness('Bags cost $5-10', ['Bags cost $5 to 10']), 1) + assert.equal(exactness('Bags cost $5-10', ['$5 - 10']), 1) + assert.equal(exactness('$10-20 seats', ['$10 for 20 seats']), 1) + assert.equal(exactness('Bags cost $5-10.', ['costs $5 and there are 10 left']), 1) + assert.equal(exactness('2024.5-2025.5%', ['2024.5% to 2025.5%']), 1) + assert.equal(exactness('2024.5-2025.5%', ['2024.5 units and 2025.5%']), 1) + assert.equal(exactness('$5-10k seconds', ['$10']), 0) + assert.equal(exactness('$5-10 EUR', ['$5 to $10']), 0.5) + }) + + // Anchors still read the bound as a bare number, so an excerpt that carries + // the figure only in its range kind cannot start contradicting a qualifier. + it('does not change which excerpts answer the qualifier checks', () => { + assert.equal(exactness('fell $5—10', ['rose $10 for 20 seats', 'costs $5 and there are 10 left']), 1) + }) +}) From c0c82987e4df4617e808c1e63a3226a69f3bbb73 Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 01:08:27 -0500 Subject: [PATCH 09/13] docs: describe the generate prompt's evidence rules as they are now The passage still quoted an `EVIDENCE_RULES` constant that said Datum has no first-party data. Describe `evidenceRules` with and without a bank, the evidence-bank section, and when the rules appear. Co-Authored-By: Claude Opus 5.5 --- docs/information-gain.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/information-gain.md b/docs/information-gain.md index 2c6e6c65..191511a3 100644 --- a/docs/information-gain.md +++ b/docs/information-gain.md @@ -38,16 +38,15 @@ Claims and clustering results are pooled up to `FACET_CLAIM_CAP` (400) claims pe ## Gap-fed generation -The research stage discovers what the baseline already covers and leaves open *before* generation, not only after scoring a draft. `gapsBlock` (`pipeline/src/generatePrompt.ts`) turns an article's `research.facets` and `research.gaps` into up to four prompt sections, inserted into `buildPrompt` right before the `# Output` instructions: +The research stage discovers what the baseline already covers and leaves open *before* generation, not only after scoring a draft. `gapsBlock` (`pipeline/src/generatePrompt.ts`) turns an article's `research.facets` and `research.gaps`, the workspace's evidence bank and any revision notes into up to five prompt sections, inserted into `buildPrompt` right before the `# Output` instructions: 1. **`# Consensus facets (must cover)`** — every facet the baseline agrees on, each annotated either `(required by template)` or `(covered by N baseline sources)` — "sources", not "ranking pages", because `docCount` counts every baseline document, our own published articles included. 2. **`# Information gaps (opportunities)`** — facets or angles the baseline leaves unanswered, each with the kind of evidence that would settle it. -3. **`# Evidence rules`** — the `EVIDENCE_RULES` constant, verbatim, whenever either of the above sections is present. -4. **`# Revision notes (previous attempt)`** — a re-run's `article.revisionNotes`, when present, ending with "Fix these before anything else." +3. **`# Evidence rules`** — `evidenceRules(companyName, hasBank)` from `cms/src/lib/tenant/evidenceBank.ts`, whenever there are facets or gaps, a company name or target domain, or a usable evidence bank. +4. **`# Evidence bank`** — the workspace's usable verified claims, facts and "never state these" entries, directly after the rules that point at it (see [tenant-context.md](tenant-context.md)). +5. **`# Revision notes (previous attempt)`** — a re-run's `article.revisionNotes`, when present, ending with "Fix these before anything else." -The evidence rules exist because "add more unique insights" without a boundary reads as an invitation to fabricate — Datum has no first-party data, so an instruction to be original has to be paired with a rule against synthetic novelty: - -> Do not invent unique insights. Add a novel factual claim only when you can name the public source (organisation and document) a fact-checker could find; otherwise state it as an explicitly labelled inference (for example, 'In our reading of the guidance…'). Never present first-party measurements, tests, surveys, or datasets — Datum has none. Prefer covering every consensus facet over adding novelty. Every number, date, and percentage must be one you can attribute. +The evidence rules exist because "add more unique insights" without a boundary reads as an invitation to fabricate. A novel factual claim needs a public source a fact-checker could find, or must be stated as a labelled inference. First-party facts (anything about the company, its product, customers, results, pricing or measurements) depend on the bank: with one, they may be stated only from its entries, each cited with its ref (`… [E3].`) and within its limits; without one, they may not be stated at all. Those cited claims are what the first-party exemption under [The policy gates](#the-policy-gates) lets through scoring. An article generated before this feature has no `facets`/`gaps` and so gets none of these sections — generation behaves exactly as it did before corpus snapshots existed. A *new* article can no longer reach generation that way: a build that yields zero claims throws in the research stage, so the article never leaves `topic_selected`. From 95e302f2e8def50798cfca9f7d2e053a40336bce Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 01:17:15 -0500 Subject: [PATCH 10/13] fix(pipeline): pass the evidence bank when reading citation markers The bank-aware marker rule had no effect in real runs: the generate stage still called the readers without the bank, so "[F5]" and "[F12]" in prose were stripped and recorded. Pass the workspace's bank to every reader and stripper, label the new evidence-bank quality source on the review page, regenerate the policy field's description, and document the rule. Co-Authored-By: Claude Opus 5.5 --- cms/src/components/ops/articleStatus.ts | 1 + .../components/ops/review/ScorecardClaims.tsx | 1 + cms/src/payload-types.ts | 2 +- .../int/informationGainRunView.int.spec.ts | 15 ++++++++ docs/tenant-context.md | 5 ++- pipeline/src/generate.ts | 35 +++++++++++-------- pipeline/test/generateStage.test.ts | 28 +++++++++++++++ 7 files changed, 71 insertions(+), 16 deletions(-) diff --git a/cms/src/components/ops/articleStatus.ts b/cms/src/components/ops/articleStatus.ts index 01508784..249b03eb 100644 --- a/cms/src/components/ops/articleStatus.ts +++ b/cms/src/components/ops/articleStatus.ts @@ -778,6 +778,7 @@ function toEvidence(value: unknown): ScorecardEvidence[] { qualityScore: num(e.qualityScore), qualitySource: e.qualitySource === 'evidence-sources' || + e.qualitySource === 'evidence-bank' || e.qualitySource === 'rubric' || e.qualitySource === 'rubric_capped' ? e.qualitySource diff --git a/cms/src/components/ops/review/ScorecardClaims.tsx b/cms/src/components/ops/review/ScorecardClaims.tsx index e12da82b..b2e120ad 100644 --- a/cms/src/components/ops/review/ScorecardClaims.tsx +++ b/cms/src/components/ops/review/ScorecardClaims.tsx @@ -7,6 +7,7 @@ import type { InformationGainRunView, ScorecardClaim } from '../articleStatus' /** How a source's quality score was arrived at — see `Evidence.qualitySource`. */ const QUALITY_SOURCE_LABEL: Record = { 'evidence-sources': 'evidence-sources table', + 'evidence-bank': 'evidence bank', rubric: 'rubric', rubric_capped: 'rubric (capped)', } diff --git a/cms/src/payload-types.ts b/cms/src/payload-types.ts index af41247f..d138d887 100644 --- a/cms/src/payload-types.ts +++ b/cms/src/payload-types.ts @@ -3077,7 +3077,7 @@ export interface InformationGainPolicy { */ requireEvidenceLineage?: ('enabled' | 'disabled') | null; /** - * Drafts are written by a model, so any claim that we ran our own test, survey, or study is fabricated. Block it. If a draft fails this check, it gets blocked from publishing. Leave this blank to fall back to the INFORMATION_GAIN_BLOCK_FIRST_PARTY_MEASUREMENTS environment variable, or the built-in default of true. + * Drafts are written by a model, so a claim that we ran our own test, survey, or study is fabricated unless it restates an entry in the workspace's evidence bank. Block every other one. If a draft fails this check, it gets blocked from publishing. Leave this blank to fall back to the INFORMATION_GAIN_BLOCK_FIRST_PARTY_MEASUREMENTS environment variable, or the built-in default of true. */ blockFirstPartyMeasurements?: ('enabled' | 'disabled') | null; /** diff --git a/cms/tests/int/informationGainRunView.int.spec.ts b/cms/tests/int/informationGainRunView.int.spec.ts index 9a72450e..447e6cfe 100644 --- a/cms/tests/int/informationGainRunView.int.spec.ts +++ b/cms/tests/int/informationGainRunView.int.spec.ts @@ -245,6 +245,21 @@ describe('toRunView — missing and malformed input', () => { expect(view.scores.verifiedGainUnits).toBeNull() }) + // A first-party claim verified against the workspace's evidence bank carries + // its own quality source; the review page must show it, not "source unknown". + it('keeps the evidence-bank quality source', () => { + const view = toRunView( + run({ + claims: [ + claim('c1', { + evidence: [{ domain: 'evidence bank', excerpt: 'E3', qualitySource: 'evidence-bank' }], + }), + ], + } as never), + ) + expect(view.claims[0]?.evidence[0]?.qualitySource).toBe('evidence-bank') + }) + it('falls back to a review decision when the stored decision is not one of the four', () => { expect(toRunView(run({ decision: 'MAYBE' } as never)).decision).toBe('HUMAN_REVIEW') }) diff --git a/docs/tenant-context.md b/docs/tenant-context.md index 66d16eba..ee300a4a 100644 --- a/docs/tenant-context.md +++ b/docs/tenant-context.md @@ -304,7 +304,10 @@ stating a first-party fact. `generateStage` finds those markers in every generated string field — title, slug, meta and OG fields, FAQ questions and answers, and the body — records `{ref, excerpt}` on `article.evidenceCitations` where the excerpt is the sentence that carried the marker, and then strips every -marker before anything is stored. No reader ever sees one. +marker before anything is stored. No reader ever sees one. `[E]` is always a +marker, so an invented `[E9]` still reaches QA as an unknown ref; `[F]` and +`[R]` count only when the bank has that ref, so "press [F5]" in ordinary +prose is left as it is. ### Expiry diff --git a/pipeline/src/generate.ts b/pipeline/src/generate.ts index 98e70ee7..8c54c8a1 100644 --- a/pipeline/src/generate.ts +++ b/pipeline/src/generate.ts @@ -4,7 +4,7 @@ import { buildPrompt, buildSystemPrompt } from './generatePrompt' import { completeJSONLogged } from './llm' import { markdownToLexical } from './richtext' import { resolveTemplate, type Stage } from './stages' -import { evidenceRefsIn, selectIcp, stripEvidenceRefs } from './tenant' +import { evidenceRefsIn, selectIcp, stripEvidenceRefs, type EvidenceRefOptions } from './tenant' interface GeneratedArticle { title: string @@ -89,15 +89,18 @@ function sentencesOf(text: string): string[] { * the bank, and the review view shows the reviewer the sentence each one is * carrying — a bare list of refs would make them hunt for it. */ -export function extractEvidenceCitations(fields: (string | null | undefined)[]): EvidenceCitation[] { +export function extractEvidenceCitations( + fields: (string | null | undefined)[], + opts?: EvidenceRefOptions, +): EvidenceCitation[] { const seen = new Set() const citations: EvidenceCitation[] = [] for (const field of fields) { if (!field) continue for (const sentence of sentencesOf(field)) { - const refs = evidenceRefsIn(sentence) + const refs = evidenceRefsIn(sentence, opts) if (refs.length === 0) continue - const excerpt = stripEvidenceRefs(sentence).replace(MARKUP_PREFIX, '').trim().slice(0, MAX_EXCERPT) + const excerpt = stripEvidenceRefs(sentence, opts).replace(MARKUP_PREFIX, '').trim().slice(0, MAX_EXCERPT) for (const ref of refs) { const key = `${ref}\u0000${excerpt}` if (seen.has(key)) continue @@ -128,6 +131,10 @@ export const generateStage: Stage = { // and a marker that survives into the stored article is a leak a reader // sees. Collect first, then strip: after stripping there is nothing left to // record. `markdownToLexical` runs on the stripped body for the same reason. + // `[E]` is always a marker; `[F]` and `[R]` only when this + // workspace's bank has that ref, so "press [F5]" in prose is left alone. + const refOptions: EvidenceRefOptions = { bank: ctx.tenant.evidenceBank } + const strip = (text: string) => stripEvidenceRefs(text, refOptions) const citations = extractEvidenceCitations([ generated.title, generated.slug, @@ -137,22 +144,22 @@ export const generateStage: Stage = { generated.ogDescription, generated.bodyMarkdown, ...generated.faqItems.flatMap((item) => [item.question, item.answer]), - ]) + ], refOptions) return { status: 'drafted', data: { - title: stripEvidenceRefs(generated.title), - slug: stripEvidenceRefs(generated.slug), - titleTag: stripEvidenceRefs(generated.titleTag), - metaDescription: stripEvidenceRefs(generated.metaDescription), - ogTitle: stripEvidenceRefs(generated.ogTitle), - ogDescription: stripEvidenceRefs(generated.ogDescription), + title: strip(generated.title), + slug: strip(generated.slug), + titleTag: strip(generated.titleTag), + metaDescription: strip(generated.metaDescription), + ogTitle: strip(generated.ogTitle), + ogDescription: strip(generated.ogDescription), ogImage: generated.ogImage, faqItems: generated.faqItems.map((item) => ({ - question: stripEvidenceRefs(item.question), - answer: stripEvidenceRefs(item.answer), + question: strip(item.question), + answer: strip(item.answer), })), - body: markdownToLexical(stripEvidenceRefs(generated.bodyMarkdown)) as Article['body'], + body: markdownToLexical(strip(generated.bodyMarkdown)) as Article['body'], evidenceCitations: citations, generationModel: result.model, }, diff --git a/pipeline/test/generateStage.test.ts b/pipeline/test/generateStage.test.ts index 7f9afc18..822b2f64 100644 --- a/pipeline/test/generateStage.test.ts +++ b/pipeline/test/generateStage.test.ts @@ -174,6 +174,34 @@ test('the generate prompt carries the bank the citations are checked against', a assert.match(requests.generate.system, /# Workspace/) }) +// `[F]` and `[R]` read naturally as prose (a keyboard key, a figure), so +// they are markers only when the bank has that ref; `[E]` always is. +test('bracketed prose that names no bank entry is left in the text and not cited', async () => { + const { ctx } = ctxWith({ + llm: { + async completeJSON() { + return { + json: { + ...JSON.parse(JSON.stringify(GENERATED)), + bodyMarkdown: 'Press [F12] to open the developer tools, then cite the catalogue [F5].', + }, + provider: 'mock', + model: 'mock-generate', + usage: { inputTokens: 0, outputTokens: 0, webSearchRequests: 0 }, + } + }, + } as unknown as LlmClient, + }) + const outcome = await generateStage.run(article, ctx) + const data = outcome.data as Record + const body = lexicalToMarkdown(data.body as RichText) + assert.match(body, /Press \[F12\] to open the developer tools/) + assert.ok(!body.includes('[F5]'), `a real bank ref survived: ${body}`) + const refs = (data.evidenceCitations as { ref: string }[]).map((c) => c.ref) + assert.ok(refs.includes('F5')) + assert.ok(!refs.includes('F12')) +}) + test('a workspace with no bank records no citations and stores the draft unchanged', async () => { const { ctx } = ctxWith({ tenant: emptyTenantContext('2026-09-02') }) const outcome = await generateStage.run(article, ctx) From 2e8d3c0e4fbf75324bc1a9918ef7c8c2c2c79185 Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 01:30:04 -0500 Subject: [PATCH 11/13] fix(pipeline): store citation excerpts as plain text Scoring matches a citation's excerpt against the body read back as plain text, so a cited sentence carrying a link or bold never matched and a bank-backed measurement stayed blocked. Read each excerpt through the shared converter before storing it. Co-Authored-By: Claude Opus 5.5 --- pipeline/src/generate.ts | 9 +++++++-- pipeline/test/generateStage.test.ts | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pipeline/src/generate.ts b/pipeline/src/generate.ts index 8c54c8a1..84543c2d 100644 --- a/pipeline/src/generate.ts +++ b/pipeline/src/generate.ts @@ -2,7 +2,7 @@ import type { Article } from '../../cms/src/payload-types' import { buildPrompt, buildSystemPrompt } from './generatePrompt' import { completeJSONLogged } from './llm' -import { markdownToLexical } from './richtext' +import { lexicalToPlainText, markdownToLexical, type RichText } from './richtext' import { resolveTemplate, type Stage } from './stages' import { evidenceRefsIn, selectIcp, stripEvidenceRefs, type EvidenceRefOptions } from './tenant' @@ -100,7 +100,12 @@ export function extractEvidenceCitations( for (const sentence of sentencesOf(field)) { const refs = evidenceRefsIn(sentence, opts) if (refs.length === 0) continue - const excerpt = stripEvidenceRefs(sentence, opts).replace(MARKUP_PREFIX, '').trim().slice(0, MAX_EXCERPT) + // As plain text, the way the stored body is read back for scoring: a + // linked or bolded sentence must still match it word for word. + const plain = lexicalToPlainText( + markdownToLexical(stripEvidenceRefs(sentence, opts)) as RichText, + ) + const excerpt = plain.replace(MARKUP_PREFIX, '').trim().slice(0, MAX_EXCERPT) for (const ref of refs) { const key = `${ref}\u0000${excerpt}` if (seen.has(key)) continue diff --git a/pipeline/test/generateStage.test.ts b/pipeline/test/generateStage.test.ts index 822b2f64..0e4fe03a 100644 --- a/pipeline/test/generateStage.test.ts +++ b/pipeline/test/generateStage.test.ts @@ -11,6 +11,7 @@ import { lexicalToMarkdown, type RichText } from '../src/richtext' import type { StageContext } from '../src/stages' import { stages } from '../src/stages' import { extractEvidenceCitations } from '../src/generate' +import { firstPartyMatches } from '../src/informationGain/passes' const generateStage = stages.find((stage) => stage.name === 'generate')! import { loadStyleGuide } from '../src/styleGuide' @@ -243,6 +244,24 @@ test('a lower-cased marker is stripped and recorded as the entry it names', asyn assert.ok(citations.some((c) => c.ref === 'F4' && c.excerpt === 'It holds.')) }) +// Scoring reads the stored body back as plain text and matches the citation's +// excerpt against it, so markdown left in the excerpt (a link, bold) would stop +// a bank-backed claim from ever matching. +test('a citation excerpt is plain text, without the markdown around it', () => { + const [citation] = extractEvidenceCitations([ + 'Our [customer study](/research) found **median** onboarding time was 11 days across 120 customers [E1].', + ]) + assert.equal( + citation?.excerpt, + 'Our customer study found median onboarding time was 11 days across 120 customers.', + ) // And so a claim extracted from the plain body matches it. + const claim = { + id: 'c001', + text: 'Our customer study found median onboarding time was 11 days across 120 customers', + } + assert.equal(firstPartyMatches([claim as never], [citation!]).get('c001'), 'E1') +}) + test('extractEvidenceCitations caps an excerpt and de-duplicates identical rows', () => { const long = `${'word '.repeat(200)}[E1]` const [only] = extractEvidenceCitations([long]) From 4feb595a56578036d14b41630c87a1caa8f1854d Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 01:44:43 -0500 Subject: [PATCH 12/13] test(cms): check hostile tags case-insensitively in the rich-text escape test Co-Authored-By: Claude Opus 5.5 --- cms/tests/int/richText.int.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/tests/int/richText.int.spec.ts b/cms/tests/int/richText.int.spec.ts index d33c1746..322421ff 100644 --- a/cms/tests/int/richText.int.spec.ts +++ b/cms/tests/int/richText.int.spec.ts @@ -107,7 +107,7 @@ describe('lexicalToHtml', () => { const out = html( '## \n **x**\n- \n> "q" & ', ) - expect(out).not.toMatch(/|/) + expect(out).not.toMatch(/|/i) expect(out).toContain('<script>alert(1)</script>') expect(out).toContain('<b>x</b>') expect(out).toContain('
    "q" & <i>
    ') From 37099c7c0d3f7ced97be9f9c80904518b00af06e Mon Sep 17 00:00:00 2001 From: snowopsdev Date: Tue, 10 Nov 2026 01:48:07 -0500 Subject: [PATCH 13/13] fix(cms): keep adjacent bold and italic readable when a template is saved again Neighbouring text nodes with different marks were written as `**Bold***italic*`, which the parser reads as literal asterisks, so a second save of a template corrupted it. Join neighbours that share marks, and write the italic side with `_` where two `*` runs would touch. Co-Authored-By: Claude Opus 5.5 --- cms/src/lib/richText.ts | 47 +++++++++++++++++++++++++----- cms/tests/int/richText.int.spec.ts | 18 ++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/cms/src/lib/richText.ts b/cms/src/lib/richText.ts index 45dba2f1..689b19f5 100644 --- a/cms/src/lib/richText.ts +++ b/cms/src/lib/richText.ts @@ -321,7 +321,7 @@ export function inlineText(node: unknown): string { return childrenOf(n).map(inlineText).join('') } -function inlineMarkdown(node: Loose): string { +function inlineMarkdown(node: Loose, italic = '*'): string { if (node.type === 'linebreak') return '\n' if (node.type === 'tab') return '\t' if (typeof node.text === 'string') { @@ -331,10 +331,10 @@ function inlineMarkdown(node: Loose): string { let marked = core if (format & FORMAT_CODE) marked = `\`${marked}\`` if (format & FORMAT_BOLD) marked = `**${marked}**` - if (format & FORMAT_ITALIC) marked = `*${marked}*` + if (format & FORMAT_ITALIC) marked = `${italic}${marked}${italic}` return lead + marked + trail } - const inner = childrenOf(node).map(inlineMarkdown).join('') + const inner = inlineRunMarkdown(childrenOf(node)) if (node.type === 'link' || node.type === 'autolink') { const url = linkUrlOf(node) return url && inner.trim() ? `[${inner}](${url})` : inner @@ -342,6 +342,39 @@ function inlineMarkdown(node: Loose): string { return inner } +/** + * Markdown for a run of sibling inline nodes. Neighbours with the same marks are + * joined first. Where two `*` delimiters would still touch (`**Bold***italic*` + * reads back as literal asterisks), the italic side is written with `_`. + */ +function inlineRunMarkdown(nodes: Loose[]): string { + const merged: Loose[] = [] + for (const node of nodes) { + const prev = merged[merged.length - 1] + const sameMarks = + prev !== undefined && + typeof prev.text === 'string' && + typeof node.text === 'string' && + prev.type === node.type && + formatOf(prev) === formatOf(node) + if (sameMarks) merged[merged.length - 1] = { ...prev, text: `${prev.text}${node.text}` } + else merged.push(node) + } + const pieces = merged.map((node) => inlineMarkdown(node)) + for (let k = 1; k < pieces.length; k++) { + if (!pieces[k - 1].endsWith('*') || !pieces[k].startsWith('*')) continue + for (const side of [k, k - 1]) { + const alt = inlineMarkdown(merged[side], '_') + // `_` only opens and closes away from letters and digits. + if (alt === pieces[side]) continue + if (isWordChar(pieces[side - 1]?.slice(-1)) || isWordChar(pieces[side + 1]?.[0])) continue + pieces[side] = alt + break + } + } + return pieces.join('') +} + const levelOf = (tag: unknown, fallback: number): number => { const level = typeof tag === 'string' ? Number.parseInt(tag.replace(/^h/, ''), 10) : Number.NaN return level >= 1 && level <= 6 ? level : fallback @@ -370,10 +403,8 @@ function listLines(list: Loose, depth: number, markdown: boolean): string[] { continue } const kids = childrenOf(item) - const text = kids - .filter((kid) => kid.type !== 'list') - .map((kid) => (markdown ? inlineMarkdown(kid) : inlineText(kid))) - .join('') + const inline = kids.filter((kid) => kid.type !== 'list') + const text = (markdown ? inlineRunMarkdown(inline) : inline.map(inlineText).join('')) .replace(/\s*\n\s*/g, ' ') .trim() if (text) { @@ -390,7 +421,7 @@ function listLines(list: Loose, depth: number, markdown: boolean): string[] { function blockLines(node: Loose, markdown: boolean): string[] { if (node.type === 'list') return listLines(node, 0, markdown) - const text = (markdown ? childrenOf(node).map(inlineMarkdown).join('') : inlineText(node)).trim() + const text = (markdown ? inlineRunMarkdown(childrenOf(node)) : inlineText(node)).trim() if (!text) return [] if (node.type === 'heading' && markdown) { return [`${'#'.repeat(levelOf(node.tag, 2))} ${text.replace(/\s*\n\s*/g, ' ')}`] diff --git a/cms/tests/int/richText.int.spec.ts b/cms/tests/int/richText.int.spec.ts index 322421ff..4056ead8 100644 --- a/cms/tests/int/richText.int.spec.ts +++ b/cms/tests/int/richText.int.spec.ts @@ -192,6 +192,24 @@ describe('template textarea round trip', () => { expect(lexicalToMarkdown(body, '\n')).toBe(text) }) + // Adjacent nodes with different marks used to come back as `**Bold***italic*`, + // which the next save read as literal asterisks. + it('keeps adjacent bold and italic apart through a save', () => { + for (const text of ['**Bold**_italic_', '_italic_**Bold** after', 'a **b**_c_ and `d`']) { + const body = plainTextToLexical(text) + expect(lexicalToMarkdownText(body)).toBe(text) + expect(plainTextToLexical(lexicalToMarkdownText(body))).toEqual(body) + } + }) + + it('writes neighbouring text nodes with the same marks as one run', () => { + const bold = (text: string) => ({ type: 'text', text, format: 1, version: 1 }) + const body = { + root: { type: 'root', children: [{ type: 'paragraph', children: [bold('Bo'), bold('ld')] }] }, + } + expect(lexicalToMarkdownText(body as never)).toBe('**Bold**') + }) + // Readers of the words (reading time, banned-phrase matching) get no markers: // a bolded half of a phrase must still match the phrase. it('gives plain text with no markdown to the readers that want words', () => {