From 8c610b895dad749fa67fedf77584e76796aa3d3a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 12:29:47 +0000 Subject: [PATCH 01/11] test(metadata-protocol,mcp,plugin-audit,service-analytics): pin the stored content-hash exit family before the fix (#21207) Pins written red-first: the served form of the stored content hash at every door, inbound version tokens in keyed form, the evaluate refusals, the write-time copies and the extended at-rest migration. Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- ...io-data-bridge.stored-content-hash.test.ts | 301 +++++++++++++++ ...ocol.data-door-stored-content-hash.test.ts | 258 +++++++++++++ .../src/protocol.served-content-hash.test.ts | 360 ++++++++++++++++++ .../plugin-audit/src/audit-writers.test.ts | 91 +++++ .../stored-metadata-hash-migration.test.ts | 177 +++++++++ .../src/stored-metadata-body-refusal.test.ts | 41 ++ 6 files changed, 1228 insertions(+) create mode 100644 packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts create mode 100644 packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts create mode 100644 packages/metadata-protocol/src/protocol.served-content-hash.test.ts create mode 100644 packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts diff --git a/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts b/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts new file mode 100644 index 00000000000..9f865f04664 --- /dev/null +++ b/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts @@ -0,0 +1,301 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21207] Exit two at the MCP stdio transport's engine-only reader: the two + * stored content-hash columns of `sys_metadata` / `sys_metadata_history` + * (`checksum`, and the history table's `previous_checksum`). + * + * The engine returns them as stored — a hash over the WHOLE stored body, + * withheld credential material included — so an administrator's key read them + * raw beside the projected body: an offline verifier for a guess at the + * withheld material. Per the maintainer's ruling, this transport now serves + * each as the crypto provider's keyed digest of the stored value (omitted when + * no provider is registered), on every read path — the bridge's query and get + * and the ADR-0101 record resource — and refuses a filter, sort or grouping on + * either column before the engine is asked (`INVALID_FIELD` / 400, the shape + * its body-column refusal already answers). A member stays refused by the + * engine, unchanged. + */ + +import { createHmac } from 'node:crypto'; +import { PassThrough } from 'node:stream'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import { SysMetadataObject, SysMetadataHistoryObject } from '@objectstack/metadata-core'; +import type { ExecutionContext } from '@objectstack/spec/kernel'; +import type { IDataEngine, IMetadataService } from '@objectstack/spec/contracts'; +import type { MCPServerRuntime } from './mcp-server-runtime.js'; +import { MCPServerPlugin } from './plugin.js'; +import { createStdioDataBridge } from './stdio-data-bridge.js'; + +const keyedDigest = async (plain: string): Promise => + `hmac-sha256:${createHmac('sha256', 'mcp-test-key').update(plain, 'utf8').digest('hex')}`; +const KEYED = /^hmac-sha256:[0-9a-f]{64}$/; + +const HASH_M = `sha256:${'1'.repeat(64)}`; +const HASH_H = `sha256:${'2'.repeat(64)}`; +const PARENT_H = `sha256:${'3'.repeat(64)}`; +const STORED = [HASH_M, HASH_H, PARENT_H]; + +const TABLES: Record>> = { + sys_metadata: [{ id: 'm1', type: 'view', name: 'v', metadata: '{"name":"v"}', checksum: HASH_M, state: 'active' }], + sys_metadata_history: [ + { id: 'h1', type: 'view', name: 'v', metadata: '{"name":"v"}', checksum: HASH_H, previous_checksum: PARENT_H }, + { id: 'h0', type: 'view', name: 'v', metadata: '{"name":"v"}', checksum: PARENT_H, previous_checksum: null }, + ], + file_blob: [{ id: 'f1', name: 'blob', checksum: HASH_M }], +}; + +const ADMIN = 'usr_admin'; +const MEMBER = 'usr_member'; + +function matches(row: Record, where: unknown): boolean { + for (const [key, cond] of Object.entries((where ?? {}) as Record)) { + if (key.startsWith('$') || (cond !== null && typeof cond === 'object')) { + throw new Error(`fixture where-matcher: unsupported shape on '${key}'`); + } + if (row[key] !== cond) return false; + } + return true; +} + +function fakeEngine(opts: { owner?: string; provider?: boolean } = {}) { + const owner = opts.owner ?? ADMIN; + const engine: Record = { + find: vi.fn(async (object: string, query?: any, options?: any) => { + const context = options?.context ?? query?.context; + if (object === 'sys_api_key') return [{ id: 'k1', user_id: owner, revoked: false }]; + if (object.startsWith('sys_metadata') && context?.userId !== ADMIN) { + throw Object.assign(new Error(`Permission denied: cannot read '${object}'`), { code: 'PERMISSION_DENIED', status: 403 }); + } + const rows = (TABLES[object] ?? []).filter((row) => matches(row, query?.where)); + return typeof query?.limit === 'number' ? rows.slice(0, query.limit).map((r) => ({ ...r })) : rows.map((r) => ({ ...r })); + }), + aggregate: vi.fn(async () => [] as unknown[]), + }; + if (opts.provider !== false) engine.getKeyedDigest = () => keyedDigest; + return engine; +} + +const DEFS: Record = { + sys_metadata: SysMetadataObject, + sys_metadata_history: SysMetadataHistoryObject, + file_blob: { name: 'file_blob', fields: { name: { type: 'text' }, checksum: { type: 'text' } } }, +}; + +function fakeMetadata() { + return { + listObjects: vi.fn(async () => Object.values(DEFS)), + getObject: vi.fn(async (name: string) => DEFS[name] ?? null), + get: vi.fn(async () => null), + list: vi.fn(async () => []), + exists: vi.fn(async () => true), + getRegisteredTypes: vi.fn(async () => ['object']), + register: vi.fn(), + unregister: vi.fn(), + }; +} + +function bridgeAs(userId: string, opts: { provider?: boolean } = {}) { + const engine = fakeEngine(opts); + const bridge = createStdioDataBridge({ + engine: engine as unknown as IDataEngine, + metadataService: fakeMetadata() as unknown as IMetadataService, + resolvePrincipal: async () => ({ userId, isSystem: false }) as unknown as ExecutionContext, + keyedDigest: () => (engine as { getKeyedDigest?: () => typeof keyedDigest }).getKeyedDigest?.(), + }); + return { bridge, engine: engine as any }; +} + +const noStoredHash = (value: unknown) => { + const text = JSON.stringify(value); + for (const s of STORED) expect(text).not.toContain(s); +}; + +describe('[#21207] stdio reader: an administrator is served the keyed hash, never the stored one', () => { + it('query: both columns keyed on both tables, stable across reads, a null parent stays null', async () => { + const { bridge } = bridgeAs(ADMIN); + const m1 = (await bridge.query('sys_metadata', {})) as { records: Array> }; + const m2 = (await bridge.query('sys_metadata', {})) as { records: Array> }; + expect(m1.records[0]!.checksum).toMatch(KEYED); + expect(m1.records[0]!.checksum).toBe(await keyedDigest(HASH_M)); + expect(m2.records[0]!.checksum).toBe(m1.records[0]!.checksum); + noStoredHash(m1); + + const h = (await bridge.query('sys_metadata_history', {})) as { records: Array> }; + const h1 = h.records.find((r) => r.id === 'h1')!; + expect(h1.checksum).toMatch(KEYED); + expect(h1.previous_checksum).toMatch(KEYED); + expect(h.records.find((r) => r.id === 'h0')!.previous_checksum).toBeNull(); + noStoredHash(h); + }); + + it('get: keyed, equal to the query', async () => { + const { bridge } = bridgeAs(ADMIN); + const row = (await bridge.get('sys_metadata', 'm1')) as Record; + expect(row.checksum).toBe(await keyedDigest(HASH_M)); + noStoredHash(row); + }); + + it('no provider: both columns are omitted', async () => { + const { bridge } = bridgeAs(ADMIN, { provider: false }); + const h = (await bridge.query('sys_metadata_history', {})) as { records: Array> }; + for (const r of h.records) { + expect('checksum' in r).toBe(false); + expect('previous_checksum' in r).toBe(false); + expect(r.name).toBe('v'); + } + const row = (await bridge.get('sys_metadata', 'm1')) as Record; + expect('checksum' in row).toBe(false); + }); + + it('control: another object keeps its own checksum column, raw', async () => { + const { bridge } = bridgeAs(ADMIN); + const res = (await bridge.query('file_blob', {})) as { records: Array> }; + expect(res.records[0]!.checksum).toBe(HASH_M); + }); +}); + +describe('[#21207] stdio reader: evaluating a content-hash column is refused (INVALID_FIELD / 400)', () => { + const CASES: Array<[string, string, (b: ReturnType['bridge']) => Promise, string]> = [ + ['query: filter', 'filter', (b) => b.query('sys_metadata', { where: { checksum: HASH_M } }), 'checksum'], + ['query: filter on the parent hash', 'filter', (b) => b.query('sys_metadata_history', { where: { previous_checksum: PARENT_H } }), 'previous_checksum'], + ['query: sort', 'sort', (b) => b.query('sys_metadata', { orderBy: [{ field: 'checksum', order: 'asc' }] }), 'checksum'], + ['aggregate: group', 'groupBy', (b) => b.aggregate!('sys_metadata', { groupBy: ['checksum'], aggregations: [{ function: 'count', alias: 'n' }] }), 'checksum'], + ['aggregate: group (object form)', 'groupBy', (b) => b.aggregate!('sys_metadata_history', { groupBy: [{ field: 'previous_checksum' }] as any, aggregations: [{ function: 'count', alias: 'n' }] }), 'previous_checksum'], + ['aggregate: filter', 'filter', (b) => b.aggregate!('sys_metadata', { where: { checksum: HASH_M }, aggregations: [{ function: 'count', alias: 'n' }] }), 'checksum'], + ]; + for (const [label, param, run, field] of CASES) { + it(`${label} → INVALID_FIELD / 400, the engine never asked`, async () => { + const { bridge, engine } = bridgeAs(ADMIN); + let caught: any; + try { + await run(bridge); + } catch (err) { + caught = err; + } + expect(caught, 'the call was refused').toBeInstanceOf(Error); + expect(caught.code).toBe('INVALID_FIELD'); + expect(caught.status).toBe(400); + expect(caught.param).toBe(param); + expect(caught.field).toBe(field); + expect(engine.find).not.toHaveBeenCalled(); + expect(engine.aggregate).not.toHaveBeenCalled(); + }); + } + + it('control: the same shapes on another object\'s checksum column are run', async () => { + const { bridge, engine } = bridgeAs(ADMIN); + await bridge.query('file_blob', { where: { checksum: HASH_M }, orderBy: [{ field: 'checksum', order: 'asc' }] }); + await bridge.aggregate!('file_blob', { groupBy: ['checksum'], aggregations: [{ function: 'count', alias: 'n' }] }); + expect(engine.find).toHaveBeenCalledTimes(1); + expect(engine.aggregate).toHaveBeenCalledTimes(1); + }); +}); + +describe('[#21207] stdio reader: a member stays refused', () => { + it('query and get answer the engine\'s PERMISSION_DENIED', async () => { + const { bridge } = bridgeAs(MEMBER); + await expect(bridge.query('sys_metadata', {})).rejects.toMatchObject({ code: 'PERMISSION_DENIED', status: 403 }); + await expect(bridge.get('sys_metadata_history', 'h1')).rejects.toMatchObject({ code: 'PERMISSION_DENIED', status: 403 }); + }); +}); + +// --------------------------------------------------------------------------- +// The ADR-0101 record resource, over the transport — the plugin's own wiring +// --------------------------------------------------------------------------- + +async function openStdio(server: any) { + const stdin = new PassThrough(); + const stdout = new PassThrough(); + const transport = new StdioServerTransport(stdin, stdout); + await server.connect(transport); + let nextId = 1; + let buffered = ''; + const waiting = new Map void>(); + stdout.on('data', (chunk: Buffer | string) => { + buffered += String(chunk); + let newline = buffered.indexOf('\n'); + while (newline >= 0) { + const line = buffered.slice(0, newline).trim(); + buffered = buffered.slice(newline + 1); + newline = buffered.indexOf('\n'); + if (!line) continue; + const frame = JSON.parse(line); + if (typeof frame.id === 'number') waiting.get(frame.id)?.(frame); + } + }); + const rpc = (method: string, params?: unknown) => + new Promise((resolve, reject) => { + const id = nextId++; + const giveUp = setTimeout(() => reject(new Error(`stdio: no answer to ${method}`)), 5_000); + waiting.set(id, (frame) => { + clearTimeout(giveUp); + resolve(frame); + }); + stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, ...(params ? { params } : {}) })}\n`); + }); + await rpc('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'content-hash-pin', version: '0.0.0' } }); + stdin.write(`${JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' })}\n`); + return { rpc, close: () => transport.close().catch(() => {}) }; +} + +async function readResource(engine: Record, uri: string): Promise> { + const registry = new Map([['metadata', fakeMetadata()], ['objectql', engine]]); + const ctx = { + registerService: vi.fn((name: string, service: unknown) => registry.set(name, service)), + getService: vi.fn((name: string) => { + if (!registry.has(name)) throw new Error(`Service "${name}" not found`); + return registry.get(name); + }), + replaceService: vi.fn(), + getServices: vi.fn(() => registry), + hook: vi.fn(), + trigger: vi.fn(async () => {}), + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + getKernel: vi.fn(() => ({})), + }; + const plugin = new MCPServerPlugin({ autoStart: true }); + await plugin.init(ctx as any); + const runtime = registry.get('mcp') as MCPServerRuntime; + vi.spyOn(runtime, 'start').mockResolvedValue(undefined); + await plugin.start(ctx as any); + const session = await openStdio(runtime.server); + try { + const frame = await session.rpc('resources/read', { uri }); + expect(frame.error, `resources/read ${uri} failed at the protocol level`).toBeUndefined(); + return JSON.parse(frame.result.contents[0].text); + } finally { + await session.close(); + } +} + +describe('[#21207] stdio record resource: the keyed hash, over the transport', () => { + const originalEnv = { ...process.env }; + beforeEach(() => { + process.env = { ...originalEnv }; + delete process.env.OS_MCP_SERVER_TRANSPORT; + delete process.env.OS_MCP_STDIO_ENABLED; + process.env.OS_MCP_STDIO_API_KEY = 'osk_stored_content_hash_pin'; + }); + afterEach(() => { + process.env = { ...originalEnv }; + vi.restoreAllMocks(); + }); + + it('administrator: both tables serve keyed values, no stored hash', async () => { + const m = await readResource(fakeEngine(), 'objectstack://objects/sys_metadata/records/m1'); + expect(m.checksum).toBe(await keyedDigest(HASH_M)); + const h = await readResource(fakeEngine(), 'objectstack://objects/sys_metadata_history/records/h1'); + expect(h.checksum).toMatch(KEYED); + expect(h.previous_checksum).toMatch(KEYED); + noStoredHash([m, h]); + }); + + it('no provider: the resource omits both columns', async () => { + const h = await readResource(fakeEngine({ provider: false }), 'objectstack://objects/sys_metadata_history/records/h1'); + expect('checksum' in h).toBe(false); + expect('previous_checksum' in h).toBe(false); + expect(h.name).toBe('v'); + }); +}); diff --git a/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts b/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts new file mode 100644 index 00000000000..e5e640fe3fa --- /dev/null +++ b/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts @@ -0,0 +1,258 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21207] Exit two at the generic data door — the two stored content-hash + * columns of `sys_metadata` / `sys_metadata_history` (`checksum`, and the + * history table's `previous_checksum`). + * + * A stored content hash is computed over the WHOLE stored body, withheld + * credential material included. Served raw beside the projected body, it is an + * offline verifier: a guess at the withheld material, hashed with the rest of + * the served body, either reproduces it or not. Evaluated as a predicate it is + * an online one. So, per the maintainer's ruling on this card: + * + * - SERVED: each column is the crypto provider's keyed digest of the stored + * value — neither the stored value nor a recomputation, stable across reads, + * moving on any body change including a credential-only one; a `null` stays + * `null`; with no provider both columns are omitted; + * - EVALUATED: a filter, sort or grouping on either column is refused before + * the engine is asked — `INVALID_FIELD` / 400, the shape the body column's + * refusals already answer — and a search never scans them (nor the body + * column): an explicit `searchFields` naming one is refused, an implicit one + * is narrowed to the other columns. + * + * Every other object — including one with its own `checksum` column — is served + * and evaluated exactly as before. + */ + +import { createHmac } from 'node:crypto'; +import { describe, expect, it, vi } from 'vitest'; +import { assertEngineFindOnePredicate, hashSpec, SysMetadataHistoryObject, SysMetadataObject } from '@objectstack/metadata-core'; +import { ObjectStackProtocolImplementation } from './protocol.js'; + +const keyedDigest = async (plain: string): Promise => + `hmac-sha256:${createHmac('sha256', 'data-door-test-key').update(plain, 'utf8').digest('hex')}`; +const KEYED = /^hmac-sha256:[0-9a-f]{64}$/; + +const CRED_A = 'data-door-cred-a'; +const CRED_B = 'data-door-cred-b'; +const dsBody = (cred: string) => ({ + name: 'edge_cache', + label: 'Edge Cache', + driver: 'turso', + config: { url: 'file:/var/data/edge.db', encryptionKey: cred }, +}); +const viewBody = { name: 'all_tasks', label: 'All Tasks', type: 'grid' }; + +const SYS_METADATA_ROWS = [ + { id: 'm_a', type: 'datasource', name: 'edge_cache', state: 'active', metadata: JSON.stringify(dsBody(CRED_A)), checksum: hashSpec(dsBody(CRED_A)) }, + { id: 'm_b', type: 'datasource', name: 'edge_cache_b', state: 'active', metadata: JSON.stringify(dsBody(CRED_B)), checksum: hashSpec(dsBody(CRED_B)) }, + { id: 'm_view', type: 'view', name: 'all_tasks', state: 'active', metadata: JSON.stringify(viewBody), checksum: hashSpec(viewBody) }, +]; +const HISTORY_ROWS = [ + { id: 'h_1', type: 'view', name: 'all_tasks', version: 1, operation_type: 'create', metadata: JSON.stringify(viewBody), checksum: hashSpec(viewBody), previous_checksum: null }, + { id: 'h_2', type: 'view', name: 'all_tasks', version: 2, operation_type: 'update', metadata: JSON.stringify(viewBody), checksum: hashSpec({ ...viewBody, label: 'x' }), previous_checksum: hashSpec(viewBody) }, +]; +/** An ordinary object with a `checksum` column of its own — not a stored metadata body. */ +const FILE_SCHEMA = { + name: 'file_blob', + fields: { name: { name: 'name', type: 'text' }, checksum: { name: 'checksum', type: 'text' } }, +}; +const FILE_ROWS = [{ id: 'f_1', name: 'blob', checksum: 'sha256:0000' }]; + +const ROWS: Record[]> = { + sys_metadata: SYS_METADATA_ROWS, + sys_metadata_history: HISTORY_ROWS, + file_blob: FILE_ROWS, +}; +const SCHEMAS: Record = { + sys_metadata: SysMetadataObject, + sys_metadata_history: SysMetadataHistoryObject, + file_blob: FILE_SCHEMA, +}; + +function matches(row: Record, where: unknown): boolean { + if (!where || typeof where !== 'object') return true; + return Object.entries(where as Record).every(([k, v]) => { + if (k.startsWith('$')) throw new Error(`stub engine: combinator '${k}' is not implemented`); + if (v !== null && typeof v === 'object') throw new Error(`stub engine: operator value on '${k}' is not implemented`); + return row[k] === v; + }); +} + +function makeProtocol(opts: { provider?: boolean } = {}) { + const find = vi.fn(async (object: string, o: any) => (ROWS[object] ?? []).filter((r) => matches(r, o?.where)).map((r) => ({ ...r }))); + const findOne = vi.fn(async (object: string, o: any) => { + assertEngineFindOnePredicate(object, o); + const hit = (ROWS[object] ?? []).find((r) => matches(r, o?.where)); + return hit ? { ...hit } : null; + }); + const aggregate = vi.fn(async () => [{ type: 'datasource', n: 2 }]); + const count = vi.fn(async () => 0); + const engine: any = { registry: { getObject: (n: string) => SCHEMAS[n] }, find, findOne, count, aggregate }; + if (opts.provider !== false) engine.getKeyedDigest = () => keyedDigest; + return { p: new ObjectStackProtocolImplementation(engine), find, findOne, aggregate, count }; +} + +const byId = (records: any[], id: string) => records.find((r) => r.id === id); + +async function refusal(run: () => Promise): Promise { + let caught: any; + let resolved = false; + try { + await run(); + resolved = true; + } catch (e) { + caught = e; + } + expect(resolved, 'expected a refusal, but the query ran').toBe(false); + return caught; +} + +describe('[#21207] data door — the content-hash columns are served keyed', () => { + it('list read: keyed, not stored, not a recomputation from the served body; stable across reads', async () => { + const { p } = makeProtocol(); + const first: any = await p.findData({ object: 'sys_metadata', query: {} }); + const second: any = await p.findData({ object: 'sys_metadata', query: {} }); + + for (const row of SYS_METADATA_ROWS) { + const served = byId(first.records, row.id); + expect(served.checksum).toMatch(KEYED); + expect(served.checksum).not.toBe(row.checksum); + const servedBody = JSON.parse(served.metadata); + expect(served.checksum).not.toBe(hashSpec(servedBody)); + expect(served.checksum).toBe(await keyedDigest(row.checksum)); + expect(byId(second.records, row.id).checksum).toBe(served.checksum); + } + }); + + it('a credential-only difference: the served bodies are identical, the served hashes are not', async () => { + const { p } = makeProtocol(); + const res: any = await p.findData({ object: 'sys_metadata', query: { type: 'datasource' } }); + const a = byId(res.records, 'm_a'); + const b = byId(res.records, 'm_b'); + const { name: _na, ...bodyA } = JSON.parse(a.metadata); + const { name: _nb, ...bodyB } = JSON.parse(b.metadata); + expect(bodyA).toEqual(bodyB); + expect(a.checksum).not.toBe(b.checksum); + expect(JSON.stringify(res.records)).not.toContain(CRED_A); + }); + + it('by-id read: keyed, equal to the list read', async () => { + const { p } = makeProtocol(); + const got: any = await p.getData({ object: 'sys_metadata', id: 'm_a' }); + const listed: any = await p.findData({ object: 'sys_metadata', query: {} }); + expect(got.record.checksum).toMatch(KEYED); + expect(got.record.checksum).toBe(byId(listed.records, 'm_a').checksum); + }); + + it('history table: both columns keyed, a null parent stays null', async () => { + const { p } = makeProtocol(); + const res: any = await p.findData({ object: 'sys_metadata_history', query: {} }); + expect(byId(res.records, 'h_1').previous_checksum).toBeNull(); + expect(byId(res.records, 'h_2').previous_checksum).toBe(await keyedDigest(HISTORY_ROWS[1]!.previous_checksum as string)); + for (const row of res.records) expect(row.checksum).toMatch(KEYED); + const got: any = await p.getData({ object: 'sys_metadata_history', id: 'h_2' }); + expect(got.record.previous_checksum).toMatch(KEYED); + }); + + it('no provider: both columns are omitted on every read, the other columns untouched', async () => { + const { p } = makeProtocol({ provider: false }); + const list: any = await p.findData({ object: 'sys_metadata', query: {} }); + for (const row of list.records) { + expect('checksum' in row).toBe(false); + expect(typeof row.name).toBe('string'); + } + const hist: any = await p.findData({ object: 'sys_metadata_history', query: {} }); + for (const row of hist.records) { + expect('checksum' in row).toBe(false); + expect('previous_checksum' in row).toBe(false); + } + const got: any = await p.getData({ object: 'sys_metadata', id: 'm_view' }); + expect('checksum' in got.record).toBe(false); + expect(got.record.name).toBe('all_tasks'); + }); + + it('control: an object outside the family keeps its own checksum column, raw', async () => { + const { p } = makeProtocol(); + const res: any = await p.findData({ object: 'file_blob', query: {} }); + expect(res.records[0].checksum).toBe('sha256:0000'); + }); +}); + +describe('[#21207] data door — the content-hash columns are never evaluated', () => { + const cases: Array<[string, string, Record, string]> = [ + ['filter', 'sys_metadata', { filter: JSON.stringify({ checksum: SYS_METADATA_ROWS[0]!.checksum }) }, 'checksum'], + ['implicit filter', 'sys_metadata', { checksum: SYS_METADATA_ROWS[0]!.checksum }, 'checksum'], + ['parent-hash filter', 'sys_metadata_history', { filter: JSON.stringify({ previous_checksum: 'x' }) }, 'previous_checksum'], + ['sort', 'sys_metadata', { sort: 'checksum' }, 'checksum'], + ['descending sort', 'sys_metadata_history', { sort: '-previous_checksum' }, 'previous_checksum'], + ['group', 'sys_metadata', { groupBy: ['checksum'], aggregations: [{ function: 'count', alias: 'n' }] }, 'checksum'], + ['group (object form)', 'sys_metadata_history', { groupBy: [{ field: 'previous_checksum' }], aggregations: [{ function: 'count', alias: 'n' }] }, 'previous_checksum'], + ['aggregation filter', 'sys_metadata', { aggregations: [{ function: 'count', alias: 'n', filter: { checksum: 'x' } }] }, 'checksum'], + ]; + for (const [label, object, query, field] of cases) { + it(`${label} on '${field}' is refused (INVALID_FIELD / 400) and the engine is never asked`, async () => { + const { p, find, aggregate, count } = makeProtocol(); + const err = await refusal(() => p.findData({ object, query })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe(field); + expect(err.object).toBe(object); + expect(find).not.toHaveBeenCalled(); + expect(aggregate).not.toHaveBeenCalled(); + expect(count).not.toHaveBeenCalled(); + }); + } + + it('the refusal names the usable columns', async () => { + const { p } = makeProtocol(); + const err = await refusal(() => p.findData({ object: 'sys_metadata', query: { sort: 'checksum' } })); + expect(err.message).toContain("'type'"); + expect(err.message).toContain("'name'"); + }); + + it('control: filter, sort and group by a scalar column still run', async () => { + const { p, find, aggregate } = makeProtocol(); + await p.findData({ object: 'sys_metadata', query: { type: 'view', sort: 'name' } }); + expect(find).toHaveBeenCalledTimes(1); + await p.findData({ object: 'sys_metadata', query: { groupBy: ['type'], aggregations: [{ function: 'count', alias: 'n' }] } }); + expect(aggregate).toHaveBeenCalledTimes(1); + }); + + it('control: an object outside the family is filterable by its own checksum column', async () => { + const { p, find } = makeProtocol(); + const res: any = await p.findData({ object: 'file_blob', query: { checksum: 'sha256:0000' } }); + expect(find).toHaveBeenCalledTimes(1); + expect(res.records).toHaveLength(1); + }); +}); + +describe('[#21207] data door — a search never scans the hash or body columns of a stored-metadata table', () => { + for (const field of ['checksum', 'metadata']) { + it(`an explicit searchFields naming '${field}' is refused (INVALID_FIELD / 400)`, async () => { + const { p, find } = makeProtocol(); + const err = await refusal(() => p.findData({ object: 'sys_metadata', query: { search: 'abc', searchFields: field } })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe(field); + expect(find).not.toHaveBeenCalled(); + }); + } + + it('an implicit search is narrowed: the engine is handed a field set without them', async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'sys_metadata_history', query: { search: 'abc' } }); + const handed = find.mock.calls[0]![1].searchFields as string[]; + expect(Array.isArray(handed)).toBe(true); + expect(handed.length).toBeGreaterThan(0); + for (const refused of ['checksum', 'previous_checksum', 'metadata']) expect(handed).not.toContain(refused); + expect(handed).toContain('name'); + }); + + it('control: a search outside the family is handed on unchanged', async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'file_blob', query: { search: 'abc' } }); + expect(find.mock.calls[0]![1].searchFields).toBeUndefined(); + }); +}); diff --git a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts new file mode 100644 index 00000000000..17ea0331728 --- /dev/null +++ b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts @@ -0,0 +1,360 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21207] Exit two — the `/meta` doors serve a stored content hash only in + * KEYED form, and compare an inbound version token in that same form. + * + * The stored content hash of a metadata body stays the canonical, unkeyed hash + * at rest (the repository's invariant is untouched). Every door that SERVES it + * — the save, publish and rollback receipts, the history read, and the 409 + * conflict refusal — serves the crypto provider's keyed digest of it instead, + * so a caller holding the projected body cannot recompute it and confirm a guess + * about withheld credential material offline. Every door that takes a version + * token back (the save door's and the reset door's optimistic lock) compares it + * in keyed form against the current stored value and hands the STORED value to + * the repository. With no provider registered nothing is served and an inbound + * token is refused: fail closed. + * + * Pinned per door, as an administrator would read it: + * - the served value is neither the stored hash nor a recomputation from the + * served body; two reads agree; a change to the body moves it; + * - the keyed token is accepted inbound, the raw stored hash is refused with + * the ADR-0112 envelope (`METADATA_CONFLICT` / 409); + * - the refusal's text and attributes carry the keyed value or none, and the + * decision-audit note it writes carries no hash at all; + * - no provider: nothing served, every inbound token refused. + * + * The engine is an in-memory double of the stored tables with the repository's + * own read and write shapes (the `protocol.lifecycle-audit-rows.test.ts` double, + * plus the engine's `getKeyedDigest` accessor), so the stored values compared + * against are the ones the real repository writes. + */ + +import { createHmac } from 'node:crypto'; +import { describe, expect, it } from 'vitest'; +import { + assertEngineDeleteDispatch, + assertEngineFindOnePredicate, + assertEngineUpdateDispatch, + hashSpec, +} from '@objectstack/metadata-core'; +import { ObjectStackProtocolImplementation } from './protocol.js'; + +const TEST_KEY = 'served-content-hash-test-key'; +/** A keyed digest with the provider contract's output shape, under a key this file holds. */ +const keyedDigest = async (plain: string): Promise => + `hmac-sha256:${createHmac('sha256', TEST_KEY).update(plain, 'utf8').digest('hex')}`; + +const SHA256 = /sha256:[0-9a-f]{64}/; +const KEYED = /^hmac-sha256:[0-9a-f]{64}$/; + +interface Row { + id: string; + type: string; + name: string; + organization_id: string | null; + package_id: string | null; + state: string; + metadata: string; + checksum?: string; + [k: string]: unknown; +} + +function keyOf(w: Record) { + return `${w.type}|${w.name}|${w.organization_id ?? '__env__'}|${w.state ?? 'active'}|${w.package_id ?? '__nopkg__'}`; +} + +function matchesWhere(r: Record, where: Record = {}): boolean { + for (const [k, v] of Object.entries(where)) { + if (k === '$or') { + if (!(v as Array>).some((c) => matchesWhere(r, c))) return false; + continue; + } + if (v === undefined) continue; + if (r[k] !== v) return false; + } + return true; +} + +function makeEngine(opts: { provider?: boolean } = {}) { + const rows = new Map(); + const historyRows: Array> = []; + const auditRows: Array> = []; + let nextId = 0; + const findRow = (w: Record): { key: string; row: Row } | null => { + if (w.id !== undefined) { + for (const [k, r] of rows) if (r.id === w.id) return { key: k, row: r }; + return null; + } + if (w.package_id !== undefined) { + const k = keyOf(w); + const r = rows.get(k); + return r ? { key: k, row: r } : null; + } + for (const [k, r] of rows) if (matchesWhere(r, w)) return { key: k, row: r }; + return null; + }; + const engine: any = { + async findOne(table: string, o: { where: Record }) { + assertEngineFindOnePredicate(table, o); + if (table === 'sys_metadata_history') return historyRows.find((h) => matchesWhere(h, o.where)) ?? null; + return findRow(o.where)?.row ?? null; + }, + async find(table: string, o: { where?: Record } = {}) { + if (table === 'sys_metadata_audit') return auditRows.filter((a) => matchesWhere(a, o.where)); + if (table === 'sys_metadata_history') return historyRows.filter((h) => matchesWhere(h, o.where)); + return Array.from(rows.values()).filter((r) => matchesWhere(r, o.where)); + }, + async insert(table: string, data: Record) { + nextId += 1; + if (table === 'sys_metadata_audit') { + auditRows.push({ id: `a_${nextId}`, ...data }); + return { id: `a_${nextId}` }; + } + if (table === 'sys_metadata_history') { + historyRows.push({ id: `h_${nextId}`, ...data }); + return { id: `h_${nextId}` }; + } + const row = { id: `r_${nextId}`, ...data } as Row; + rows.set(keyOf(data), row); + return { id: row.id }; + }, + async update(_t: string, data: Record, o: { where: Record }) { + assertEngineUpdateDispatch(data, o); + const found = findRow(o.where); + if (!found) return { id: null }; + const merged = { ...found.row, ...data } as Row; + rows.delete(found.key); + rows.set(keyOf(merged), merged); + return { id: found.row.id }; + }, + async delete(_t: string, o: { where: Record }) { + assertEngineDeleteDispatch(o); + const found = findRow(o.where); + if (!found) return { deleted: 0 }; + rows.delete(found.key); + return { deleted: 1 }; + }, + async transaction(cb: (ctx: any, info: { owned: boolean }) => Promise): Promise { + return cb(undefined, { owned: true }); + }, + registry: { registerItem: () => {}, registerObject: () => {}, getPackage: () => undefined }, + }; + if (opts.provider !== false) engine.getKeyedDigest = () => keyedDigest; + return { engine, rows, historyRows, auditRows }; +} + +const viewBody = (label: string) => ({ + name: 'case_grid', + type: 'grid', + label, + columns: ['id', 'title'], + object: 'case', + viewKind: 'list', +}); +const ORG = 'org_alpha'; +const ref = { type: 'view', name: 'case_grid', organizationId: ORG, actor: 'admin' } as const; + +/** Every stored content hash the double holds, active rows and history alike. */ +function storedHashes(h: ReturnType): Set { + const out = new Set(); + for (const r of h.rows.values()) if (typeof r.checksum === 'string') out.add(r.checksum); + for (const r of h.historyRows) { + if (typeof r.checksum === 'string') out.add(r.checksum); + if (typeof r.previous_checksum === 'string') out.add(r.previous_checksum); + } + return out; +} + +function activeHash(h: ReturnType): string { + const row = [...h.rows.values()].find((r) => r.name === 'case_grid' && r.state === 'active'); + return String(row?.checksum); +} + +async function rejection(run: () => Promise): Promise { + let caught: any; + let resolved = false; + try { + await run(); + resolved = true; + } catch (e) { + caught = e; + } + expect(resolved, 'expected a refusal, but the call resolved').toBe(false); + return caught; +} + +/** No stored hash, in either the message or any attribute of a refusal. */ +function expectNoStoredHash(value: unknown, stored: Set): void { + const text = typeof value === 'string' ? value : JSON.stringify({ + ...(value as object), + message: (value as { message?: string })?.message, + }); + for (const s of stored) expect(text).not.toContain(s); + expect(text).not.toMatch(SHA256); +} + +describe('[#21207] /meta receipts serve the keyed form of the stored content hash', () => { + it('save receipt: keyed, not the stored hash, not a recomputation; stable; moves on change', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + + const first: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const stored1 = activeHash(h); + expect(first.version).toMatch(KEYED); + expect(first.version).not.toBe(stored1); + expect(first.version).not.toBe(hashSpec(viewBody('v1'))); + expect(first.version).toBe(await keyedDigest(stored1)); + + // An identical re-save writes nothing and serves the same value. + const again: any = await p.saveMetaItem({ ...ref, item: viewBody('v1'), parentVersion: first.version } as any); + expect(again.version).toBe(first.version); + + const changed: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: first.version } as any); + expect(changed.version).toMatch(KEYED); + expect(changed.version).not.toBe(first.version); + expect(changed.version).not.toBe(activeHash(h)); + }); + + it('publish and rollback receipts: keyed, never the stored hash', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + + await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + await p.saveMetaItem({ ...ref, item: viewBody('staged'), mode: 'draft' } as any); + const published: any = await p.publishMetaItem({ ...ref } as any); + expect(published.version).toMatch(KEYED); + expect(published.version).toBe(await keyedDigest(activeHash(h))); + expect(storedHashes(h).has(published.version)).toBe(false); + + const rolled: any = await p.rollbackMetaItem({ ...ref, toVersion: 1 } as any); + expect(rolled.version).toMatch(KEYED); + expect(rolled.version).toBe(await keyedDigest(activeHash(h))); + expect(storedHashes(h).has(rolled.version)).toBe(false); + }); +}); + +describe('[#21207] the history read serves keyed hashes per event', () => { + it('every event hash and parent hash is keyed, none is stored, and two reads agree', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + + const read1 = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + const read2 = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + expect(read1.events.length).toBeGreaterThanOrEqual(2); + for (const ev of read1.events) { + expect(ev.hash).toMatch(KEYED); + if (ev.parentHash !== null) expect(ev.parentHash).toMatch(KEYED); + } + expect(read1.events.some((ev) => ev.parentHash !== null)).toBe(true); + expectNoStoredHash(JSON.stringify(read1), storedHashes(h)); + expect(JSON.stringify(read2)).toBe(JSON.stringify(read1)); + // The head event's keyed hash IS the token the save receipt served. + expect(read1.events.map((e) => e.hash)).toContain(v1.version); + }); +}); + +describe('[#21207] inbound version tokens are compared in keyed form', () => { + it('save door: the served token is accepted, the raw stored hash is refused (METADATA_CONFLICT / 409)', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const raw = activeHash(h); + + const refused = await rejection(() => + p.saveMetaItem({ ...ref, item: viewBody('raw'), parentVersion: raw } as any)); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + expect(activeHash(h)).toBe(raw); + + const accepted: any = await p.saveMetaItem({ ...ref, item: viewBody('keyed'), parentVersion: v1.version } as any); + expect(accepted.success).toBe(true); + expect(activeHash(h)).not.toBe(raw); + }); + + it('save door: a stale keyed token is refused, and the refusal carries the keyed value or none', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const v2: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + + const stale = await rejection(() => + p.saveMetaItem({ ...ref, item: viewBody('lost'), parentVersion: v1.version } as any)); + expect(stale.code).toBe('METADATA_CONFLICT'); + expect(stale.status).toBe(409); + expectNoStoredHash(stale, storedHashes(h)); + for (const attr of [stale.expectedParent, stale.actualHead]) { + if (attr !== undefined && attr !== null) expect(attr).toMatch(KEYED); + } + expect(stale.actualHead).toBe(v2.version); + }); + + it('the decision-audit note of a refused write names no hash, stored or keyed', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + await rejection(() => p.saveMetaItem({ ...ref, item: viewBody('lost'), parentVersion: v1.version } as any)); + + const denials = h.auditRows.filter((a) => a.code === 'metadata_conflict'); + expect(denials).toHaveLength(1); + const note = String(denials[0]!.note); + expect(note).not.toMatch(SHA256); + expect(note).not.toMatch(/hmac-sha256:/); + expectNoStoredHash(note, storedHashes(h)); + }); + + it('reset door: the served token is accepted, the raw stored hash is refused', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + + const refused = await rejection(() => + p.deleteMetaItem({ ...ref, parentVersion: activeHash(h) } as any)); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + expectNoStoredHash(refused, storedHashes(h)); + + const reset: any = await p.deleteMetaItem({ ...ref, parentVersion: v1.version } as any); + expect(reset.success).toBe(true); + }); +}); + +describe('[#21207] no crypto provider: nothing served, every inbound token refused (fail closed)', () => { + it('receipts and history serve no hash; an unconditional save still writes', async () => { + const h = makeEngine({ provider: false }); + const p = new ObjectStackProtocolImplementation(h.engine); + const saved: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + expect(saved.success).toBe(true); + expect(saved.version).toBe(''); + await p.saveMetaItem({ ...ref, item: viewBody('v2') } as any); + + const { events } = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + expect(events.length).toBeGreaterThanOrEqual(2); + for (const ev of events) { + expect(ev.hash).toBeNull(); + expect(ev.parentHash).toBeNull(); + } + expectNoStoredHash(JSON.stringify({ saved, events }), storedHashes(h)); + }); + + it('an inbound token — raw or not — is refused on both doors, and the refusal carries no hash', async () => { + const h = makeEngine({ provider: false }); + const p = new ObjectStackProtocolImplementation(h.engine); + await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const raw = activeHash(h); + + for (const run of [ + () => p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: raw } as any), + () => p.deleteMetaItem({ ...ref, parentVersion: raw } as any), + ]) { + const refused = await rejection(run); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + expectNoStoredHash(refused, storedHashes(h)); + } + expect(activeHash(h)).toBe(raw); + }); +}); diff --git a/packages/plugins/plugin-audit/src/audit-writers.test.ts b/packages/plugins/plugin-audit/src/audit-writers.test.ts index 919ec5fb60e..3c1d797fdc3 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.test.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.test.ts @@ -2273,3 +2273,94 @@ describe('[#21120] stored metadata body copies are redacted at write time', () = expect(JSON.parse(audit!.row.new_value).metadata).toBe(clean); }); }); + +/** + * [#21207] Exit two, fork three: a COPY never carries the stored content hash. + * + * The ledger snapshot and diff, and the activity copy, of a `sys_metadata` / + * `sys_metadata_history` write used to copy the row's `checksum` (and the + * history row's `previous_checksum`) whole — a hash over the stored body, + * withheld credential material included, i.e. an offline verifier, at rest and + * served to every ledger reader. The copy now drops both columns; the history + * table itself remains the lineage. Every other column, and every other + * object's `checksum`, is copied as before. + */ +describe('[#21207] stored metadata copies carry no content hash', () => { + const HASH = `sha256:${'a'.repeat(64)}`; + const PARENT = `sha256:${'b'.repeat(64)}`; + const NEXT = `sha256:${'c'.repeat(64)}`; + const SCHEMAS = { + ...SINGLE_TENANT, + sys_metadata: ['id', 'name', 'type', 'scope', 'metadata', 'checksum'], + sys_metadata_history: ['id', 'name', 'type', 'metadata', 'checksum', 'previous_checksum'], + file_blob: ['id', 'name', 'checksum'], + }; + const view = (label: string) => JSON.stringify({ name: 'v', type: 'grid', label }); + const hashFree = (text: string) => { + expect(text).not.toContain(HASH); + expect(text).not.toContain(PARENT); + expect(text).not.toContain(NEXT); + expect(text).not.toMatch(/\\?"(previous_)?checksum\\?"/); + }; + + it('a sys_metadata create: neither the ledger snapshot nor the activity copy carries the hash', async () => { + const { engine, fire, created } = makeEngine(SCHEMAS); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterInsert', { + object: 'sys_metadata', + input: { id: 'meta-1' }, + result: { id: 'meta-1', name: 'v', type: 'view', scope: 'platform', metadata: view('one'), checksum: HASH }, + session: { userId: 'admin-1' }, + }); + const audit = created.find((c) => c.object === 'sys_audit_log'); + const activity = created.find((c) => c.object === 'sys_activity'); + hashFree(JSON.stringify(audit!.row)); + hashFree(JSON.stringify(activity!.row)); + // The copy still records the change: the other columns survive. + const newValue = JSON.parse(audit!.row.new_value); + expect(newValue.type).toBe('view'); + expect(newValue.metadata).toBe(view('one')); + }); + + it('a sys_metadata update: the diff carries the body change and not the hash change', async () => { + const { engine, fire, created } = makeEngine(SCHEMAS); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterUpdate', { + object: 'sys_metadata', + input: { id: 'meta-1', data: { metadata: view('two') } }, + previous: { id: 'meta-1', name: 'v', type: 'view', metadata: view('one'), checksum: HASH }, + result: { id: 'meta-1', name: 'v', type: 'view', metadata: view('two'), checksum: NEXT }, + session: { userId: 'admin-1' }, + }); + const audit = created.find((c) => c.object === 'sys_audit_log'); + expect(audit).toBeDefined(); + hashFree(JSON.stringify(audit!.row)); + expect(JSON.parse(audit!.row.new_value).metadata).toBe(view('two')); + hashFree(JSON.stringify(created.find((c) => c.object === 'sys_activity')!.row)); + }); + + it('a sys_metadata_history append: neither hash column is copied', async () => { + const { engine, fire, created } = makeEngine(SCHEMAS); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterInsert', { + object: 'sys_metadata_history', + input: { id: 'h-1' }, + result: { id: 'h-1', name: 'v', type: 'view', metadata: view('one'), checksum: NEXT, previous_checksum: PARENT }, + session: { userId: 'admin-1' }, + }); + for (const c of created) hashFree(JSON.stringify(c.row)); + }); + + it('control: another object keeps its own checksum column in the copy', async () => { + const { engine, fire, created } = makeEngine(SCHEMAS); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterInsert', { + object: 'file_blob', + input: { id: 'f-1' }, + result: { id: 'f-1', name: 'blob', checksum: HASH }, + session: { userId: 'admin-1' }, + }); + const audit = created.find((c) => c.object === 'sys_audit_log'); + expect(JSON.parse(audit!.row.new_value).checksum).toBe(HASH); + }); +}); diff --git a/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts new file mode 100644 index 00000000000..afc85fc5106 --- /dev/null +++ b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts @@ -0,0 +1,177 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21207] Exit two, fork three, at rest: the copies already written carry no + * stored content hash after `os migrate audit-metadata-bodies` runs. + * + * Before this card the audit writer copied a `sys_metadata` / + * `sys_metadata_history` row's `checksum` (and the history row's + * `previous_checksum`) into the ledger snapshot / diff and the activity copy, + * and the protocol wrote both hashes of a refused optimistic-lock write into + * the decision-audit note (`sys_metadata_audit.note`) — which the writer then + * copied into the ledger again. The writers no longer do; the same operator-run + * migration that withholds the copied body now also drops the two columns from + * those copies and withholds the hashes in those notes. Dry run by default, + * idempotent, and the history table stays the lineage. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { assertEngineFindOnePredicate, assertEngineUpdateDispatch } from '@objectstack/metadata-core'; +import { + migrateStoredMetadataBodyCopies, + planActivityRowPatch, + planAuditRowPatch, + planDecisionNotePatch, + type StoredMetadataBodyMigrationEngine, +} from './stored-metadata-body-migration.js'; + +const HASH = `sha256:${'a'.repeat(64)}`; +const PARENT = `sha256:${'b'.repeat(64)}`; +const SHA256 = /sha256:[0-9a-f]{64}/; +const HASH_KEY = /"(previous_)?checksum"/; + +const view = (label: string) => JSON.stringify({ name: 'v', type: 'grid', label }); +const metaSnapshot = () => ({ id: 'm1', name: 'v', type: 'view', scope: 'platform', metadata: view('one'), checksum: HASH }); +const historySnapshot = () => ({ id: 'h1', name: 'v', type: 'view', metadata: view('one'), checksum: HASH, previous_checksum: PARENT }); +const conflictNote = `expected parent ${PARENT} but current is ${HASH}`; +const WITHHELD_NOTE = 'expected parent [withheld] but current is [withheld]'; + +describe('planAuditRowPatch / planActivityRowPatch — the hash columns leave the copy', () => { + it('a sys_metadata create snapshot loses its checksum, and nothing else', () => { + const row = { id: 'a1', object_name: 'sys_metadata', record_id: 'm1', new_value: JSON.stringify(metaSnapshot()), old_value: null }; + const patch = planAuditRowPatch(row); + expect(patch).not.toBeNull(); + const rewritten = JSON.parse(String(patch!.new_value)); + expect(rewritten).toEqual({ id: 'm1', name: 'v', type: 'view', scope: 'platform', metadata: view('one') }); + }); + + it('a sys_metadata_history snapshot loses both hash columns', () => { + const row = { id: 'a2', object_name: 'sys_metadata_history', record_id: 'h1', new_value: JSON.stringify(historySnapshot()), old_value: null }; + const patch = planAuditRowPatch(row); + expect(String(patch!.new_value)).not.toMatch(SHA256); + expect(String(patch!.new_value)).not.toMatch(HASH_KEY); + }); + + it('an update diff loses the hash on both sides', () => { + const row = { + id: 'a3', + object_name: 'sys_metadata', + record_id: 'm1', + old_value: JSON.stringify({ metadata: view('one'), checksum: PARENT }), + new_value: JSON.stringify({ metadata: view('two'), checksum: HASH }), + }; + const patch = planAuditRowPatch(row, 'view'); + expect(JSON.stringify(patch)).not.toMatch(SHA256); + expect(JSON.parse(String(patch!.new_value)).metadata).toBe(view('two')); + }); + + it('the activity pair loses the hash on both halves', () => { + const row = { id: 'ac1', object_name: 'sys_metadata_history', record_id: 'h1', metadata: JSON.stringify({ old: null, new: historySnapshot() }) }; + const patch = planActivityRowPatch(row); + expect(patch!.metadata).not.toMatch(SHA256); + expect(JSON.parse(patch!.metadata).new.name).toBe('v'); + }); + + it('is idempotent, and leaves another object\'s checksum column alone', () => { + const row = { id: 'a1', object_name: 'sys_metadata', record_id: 'm1', new_value: JSON.stringify(metaSnapshot()), old_value: null }; + const once = planAuditRowPatch(row)!; + expect(planAuditRowPatch({ ...row, new_value: once.new_value })).toBeNull(); + const foreign = { id: 'a9', object_name: 'file_blob', record_id: 'f1', new_value: JSON.stringify({ id: 'f1', checksum: HASH }), old_value: null }; + expect(planAuditRowPatch(foreign)).toBeNull(); + }); +}); + +describe('planDecisionNotePatch — the decision-audit note and its copies', () => { + it('withholds both hashes of a conflict note', () => { + const patch = planDecisionNotePatch('sys_metadata_audit', { id: 'd1', code: 'metadata_conflict', note: conflictNote }); + expect(patch).toEqual({ note: WITHHELD_NOTE }); + }); + + it('keeps a null side as null', () => { + const patch = planDecisionNotePatch('sys_metadata_audit', { id: 'd2', code: 'metadata_conflict', note: `expected parent null but current is ${HASH}` }); + expect(patch).toEqual({ note: 'expected parent null but current is [withheld]' }); + }); + + it('leaves every other note alone, and is idempotent', () => { + expect(planDecisionNotePatch('sys_metadata_audit', { id: 'd3', code: 'ok', note: 'restored from version 2' })).toBeNull(); + expect(planDecisionNotePatch('sys_metadata_audit', { id: 'd4', code: 'metadata_conflict', note: WITHHELD_NOTE })).toBeNull(); + }); + + it('rewrites the ledger and activity copies of a conflict note', () => { + const snapshot = { id: 'd1', type: 'view', name: 'v', operation: 'save', outcome: 'denied', code: 'metadata_conflict', note: conflictNote }; + const audit = planDecisionNotePatch('sys_audit_log', { + id: 'a5', object_name: 'sys_metadata_audit', record_id: 'd1', new_value: JSON.stringify(snapshot), old_value: null, + }); + expect(String(audit!.new_value)).not.toMatch(SHA256); + expect(JSON.parse(String(audit!.new_value)).note).toBe(WITHHELD_NOTE); + const activity = planDecisionNotePatch('sys_activity', { + id: 'ac5', object_name: 'sys_metadata_audit', record_id: 'd1', metadata: JSON.stringify({ old: null, new: snapshot }), + }); + expect(String(activity!.metadata)).not.toMatch(SHA256); + }); +}); + +describe('migrateStoredMetadataBodyCopies — the content-hash copies (driven)', () => { + function fakeEngine(rows: Record[]>) { + const updates: Array<{ object: string; id: string; data: Record }> = []; + const engine: StoredMetadataBodyMigrationEngine = { + async find(object) { + return rows[object] ?? []; + }, + async findOne(object, query) { + assertEngineFindOnePredicate(object, query); + const id = (query as { where?: { id?: unknown } } | undefined)?.where?.id; + return (rows[object] ?? []).find((r) => r.id === id) ?? null; + }, + async update(object, data, options) { + assertEngineUpdateDispatch(data, options === undefined ? undefined : { ...options }); + const { id, ...rest } = data as Record; + updates.push({ object, id: String(id), data: rest }); + const target = (rows[object] ?? []).find((r) => r.id === id); + if (target) Object.assign(target, rest); + return { id, ...rest }; + }, + }; + return { engine, updates }; + } + const logger = { info: vi.fn(), warn: vi.fn() }; + const fixture = () => ({ + sys_metadata: [{ id: 'm1', type: 'view', name: 'v' }], + sys_audit_log: [ + { id: 'a1', object_name: 'sys_metadata', record_id: 'm1', new_value: JSON.stringify(metaSnapshot()), old_value: null }, + { id: 'a2', object_name: 'sys_metadata_audit', record_id: 'd1', new_value: JSON.stringify({ id: 'd1', code: 'metadata_conflict', note: conflictNote }), old_value: null }, + { id: 'a3', object_name: 'file_blob', record_id: 'f1', new_value: JSON.stringify({ id: 'f1', checksum: HASH }), old_value: null }, + ], + sys_activity: [{ id: 'ac1', object_name: 'sys_metadata_history', record_id: 'h1', metadata: JSON.stringify({ old: null, new: historySnapshot() }) }], + sys_metadata_audit: [ + { id: 'd1', code: 'metadata_conflict', note: conflictNote }, + { id: 'd2', code: 'ok', note: 'active' }, + ], + }); + + it('dry run counts every copy carrying a hash, per table, and writes nothing', async () => { + const { engine, updates } = fakeEngine(fixture()); + const report = await migrateStoredMetadataBodyCopies(engine, logger, { apply: false }); + expect(updates).toHaveLength(0); + expect(report.byObject.sys_audit_log!.rewritten).toBe(2); + expect(report.byObject.sys_activity!.rewritten).toBe(1); + expect(report.byObject.sys_metadata_audit!.rewritten).toBe(1); + expect(report.rewritten).toBe(4); + }); + + it('--apply rewrites them, the written values carry no hash, and a second run finds nothing', async () => { + const rows = fixture(); + const { engine, updates } = fakeEngine(rows); + const first = await migrateStoredMetadataBodyCopies(engine, logger, { apply: true }); + expect(first.failures).toBe(0); + expect(updates.map((u) => `${u.object}/${u.id}`).sort()).toEqual([ + 'sys_activity/ac1', 'sys_audit_log/a1', 'sys_audit_log/a2', 'sys_metadata_audit/d1', + ]); + for (const u of updates) expect(JSON.stringify(u.data)).not.toMatch(SHA256); + // Another object's checksum column was not touched. + expect(String(rows.sys_audit_log[2]!.new_value)).toContain(HASH); + + const second = await migrateStoredMetadataBodyCopies(engine, logger, { apply: false }); + expect(second.rewritten).toBe(0); + }); +}); diff --git a/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts b/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts index ef9030b4475..99e88113d76 100644 --- a/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts +++ b/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts @@ -70,3 +70,44 @@ describe('storedMetadataBodyAnalyticsRefusal', () => { expect(err?.field).toBe('metadata'); }); }); + +/** + * [#21207] The two stored content-hash columns of the same tables (`checksum`, + * and the history table's `previous_checksum`). Each is a hash over the WHOLE + * stored body, withheld credential material included, so a dimension serves an + * offline verifier and a filter is an online one. They are refused in the same + * envelope as the body column, in either role, on both tables. + */ +describe('storedMetadataBodyAnalyticsRefusal — the content-hash columns (#21207)', () => { + const HASH_COLUMNS: Array<[string, string]> = [ + ['sys_metadata', 'checksum'], + ['sys_metadata_history', 'checksum'], + ['sys_metadata_history', 'previous_checksum'], + ]; + for (const [object, column] of HASH_COLUMNS) { + it(`refuses '${column}' on ${object} as a dimension or measure (INVALID_FIELD / 400, param dimensions)`, () => { + const err = storedMetadataBodyAnalyticsRefusal([field(object, column, 'aggregate')]) as any; + expect(err).toBeInstanceOf(Error); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe(column); + expect(err.object).toBe(object); + expect(err.param).toBe('dimensions'); + // The refusal names the usable columns. + expect(err.message).toContain("'type'"); + }); + + it(`refuses '${column}' on ${object} as a filter or sort (param where)`, () => { + const err = storedMetadataBodyAnalyticsRefusal([field(object, column, 'predicate')]) as any; + expect(err?.code).toBe('INVALID_FIELD'); + expect(err?.status).toBe(400); + expect(err?.field).toBe(column); + expect(err?.param).toBe('where'); + }); + } + + it('leaves a `checksum` column on an object outside the family alone', () => { + expect(storedMetadataBodyAnalyticsRefusal([field('file_blob', 'checksum', 'aggregate')])).toBeUndefined(); + expect(storedMetadataBodyAnalyticsRefusal([field('file_blob', 'previous_checksum', 'predicate')])).toBeUndefined(); + }); +}); From 63cd3569f09c0a05ea51502f94b81cb7160c2f86 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 12:40:55 +0000 Subject: [PATCH 02/11] fix(metadata-protocol,objectql)!: the /meta doors and the data door serve the stored content hash only in keyed form (#21207) The engine gains a read accessor for the registered crypto provider's keyed digest. The protocol serves keyedDigest(stored) on the save, publish, batch-publish and rollback receipts, the history read and the data door's two stored-metadata tables; compares inbound version tokens in keyed form on the save and reset doors; answers conflicts with keyed values or none; writes a hash-free decision-audit note; and refuses filter, sort, group and search over the hash columns (and search over the body column) before the engine. Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .../src/metadata-redaction.ts | 202 +++++++++ .../src/protocol.lifecycle-audit-rows.test.ts | 5 +- .../src/protocol.served-content-hash.test.ts | 3 +- packages/metadata-protocol/src/protocol.ts | 406 +++++++++++++++--- packages/objectql/src/engine.ts | 20 + 5 files changed, 569 insertions(+), 67 deletions(-) diff --git a/packages/metadata-protocol/src/metadata-redaction.ts b/packages/metadata-protocol/src/metadata-redaction.ts index 9d7d8783ec7..f26f6ac5e45 100644 --- a/packages/metadata-protocol/src/metadata-redaction.ts +++ b/packages/metadata-protocol/src/metadata-redaction.ts @@ -774,3 +774,205 @@ export function storedMetadataBodyPredicateRefusal( if (namesBody(opts.sortFields)) return make('sort', 'sort'); return undefined; } + +// --------------------------------------------------------------------------- +// The stored CONTENT HASH of the same rows: served keyed, never evaluated (#21207) +// --------------------------------------------------------------------------- + +/** + * [#21207] The columns of a {@link isStoredMetadataBodyObject} table that hold + * a stored CONTENT HASH of the body: `checksum` (both tables) and the history + * table's `previous_checksum` (the parent's hash). + * + * The hash is the canonical SHA-256 of the WHOLE stored body — withheld + * credential material included — and it stays that at rest: the repository's + * canonical-hashing invariant, the optimistic lock and the parent links are + * untouched (maintainer ruling B on #21207). What changes is what a caller is + * given. Served raw beside the projected body, it is an OFFLINE VERIFIER: a + * guess at the withheld material, hashed together with the served rest of the + * body, reproduces it exactly when the guess is right. Evaluated as a + * predicate, it is an online one. So every door that serves it serves the + * crypto provider's KEYED digest of the stored value ({@link servedContentHash}), + * every door that takes a version token back compares it in that same form, and + * a filter, sort, grouping or search on it is refused + * ({@link storedMetadataHashEvaluateRefusal}, {@link storedMetadataSearchRefusal}). + * + * Kept beside the body column's family primitives' consumer rather than in + * `@objectstack/spec/kernel` with them: that is the family's natural home, but + * outside the surface this card claimed. The surfaces that cannot import this + * package (`@objectstack/mcp`, `@objectstack/plugin-audit`, + * `@objectstack/service-analytics`) each name the same two columns, and + * `stored-metadata-body-family.pin.test.ts` pins this list to the columns the + * two object definitions declare. + */ +export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); + +/** The keyed-digest primitive of the registered crypto provider (`ICryptoProvider.keyedDigest`). */ +export type StoredHashDigest = (plain: string) => Promise; + +/** + * The form a stored content hash is SERVED in: the keyed digest of the stored + * value under the provider's server-held key; `null` when nothing is stored + * (a delete event, a first version's parent); `undefined` — WITHHELD — when no + * provider is registered, or when the stored value is not a string this + * function can judge. ⛔ Never the stored value itself. + * + * A failing digest is not caught: a provider that cannot compute it fails the + * read rather than serving what it exists to replace. + */ +export async function servedContentHash( + stored: unknown, + digest: StoredHashDigest | undefined, +): Promise { + if (stored === null || stored === undefined) return null; + if (typeof stored !== 'string' || !digest) return undefined; + return digest(stored); +} + +/** + * Serve one row of a stored-metadata table with its content-hash columns in + * their served form ({@link servedContentHash}): keyed, `null` kept `null`, and + * the column OMITTED when the value is withheld — and, with no crypto provider, + * both columns omitted outright. A row of any other object, and a row carrying + * neither column, is returned by reference. + */ +export async function serveStoredMetadataHashColumns( + object: string, + row: T, + digest: StoredHashDigest | undefined, +): Promise { + if (!isStoredMetadataBodyObject(object) || !isPlainRecord(row)) return row; + if (!STORED_METADATA_HASH_COLUMNS.some((column) => column in row)) return row; + const out: Record = { ...row }; + for (const column of STORED_METADATA_HASH_COLUMNS) { + if (!(column in out)) continue; + // No provider: the column is not served at all — a `null` included, so + // a reader cannot tell a withheld hash from an absent one either. + const served = digest ? await servedContentHash(out[column], digest) : undefined; + if (served === undefined) delete out[column]; + else out[column] = served; + } + return out as T; +} + +/** {@link serveStoredMetadataHashColumns} over the rows of one read. Non-array input passes through. */ +export async function serveStoredMetadataHashColumnRows( + object: string, + rows: T[], + digest: StoredHashDigest | undefined, +): Promise { + if (!Array.isArray(rows) || !isStoredMetadataBodyObject(object)) return rows; + return Promise.all(rows.map((row) => serveStoredMetadataHashColumns(object, row, digest))); +} + +/** The content-hash column a field reference reaches — the column, or a dotted path headed by it. */ +function hashColumnOf(field: unknown): string | undefined { + if (typeof field !== 'string') return undefined; + const head = field.split('.')[0] as string; + return STORED_METADATA_HASH_COLUMNS.includes(head) ? head : undefined; +} + +/** The columns a refusal on these tables points the caller at instead. */ +const USABLE_COLUMNS = `'${STORED_TYPE_COLUMN}', 'name', 'state' or another scalar column`; + +/** + * [#21207] The data door's refusal to EVALUATE a content-hash column of a + * stored-metadata table — a grouping (whose keys would serve the stored + * values), a filter (an online verifier: a guessed hash matches exactly one + * row) or a sort (an order over the same values) — or `undefined` when none is + * named. Maintainer ruling A on #21207's second execution fork. + * + * The family's existing body-column refusals' shape, extended to these columns + * rather than a second dialect: `INVALID_FIELD` / 400 naming the field, the + * object and the offending `param`, judged in the data door's order — grouping, + * then filter, then sort — and naming the columns that remain usable. A dotted + * path headed by a hash column is caught too. + */ +export function storedMetadataHashEvaluateRefusal( + object: string, + opts: { groupBy?: unknown; filterFields?: readonly unknown[]; sortFields?: readonly unknown[] }, +): Error | undefined { + if (!isStoredMetadataBodyObject(object)) return undefined; + const make = (param: 'groupBy' | 'filter' | 'sort', column: string, position?: string): Error => { + const doing = param === 'groupBy' ? 'group' : param; + const why = param === 'filter' + ? 'a filter on it compares a guess against the stored hash row by row, which confirms the guess' + : param === 'groupBy' + ? 'a group key would serve the stored value itself' + : 'a sort on it orders by the stored values'; + const err: any = new Error( + `Cannot ${doing} '${object}' by '${column}' (${position ?? param}): the query was not run. The ` + + `'${column}' column holds the stored content hash of a metadata body, computed over withheld ` + + `credential material too, so this door serves it only in keyed form; ${why}. ` + + `${doing === 'group' ? 'Group' : doing === 'filter' ? 'Filter' : 'Sort'} by ${USABLE_COLUMNS} instead.`, + ); + err.code = 'INVALID_FIELD'; + err.status = 400; + err.field = column; + err.fields = [column]; + err.object = object; + err.param = param; + return err; + }; + if (Array.isArray(opts.groupBy)) { + for (let i = 0; i < opts.groupBy.length; i += 1) { + const entry = opts.groupBy[i]; + const objectForm = isPlainRecord(entry); + const column = hashColumnOf(objectForm ? entry.field : entry); + if (column) return make('groupBy', column, objectForm ? `groupBy[${i}].field` : `groupBy[${i}]`); + } + } + for (const field of opts.filterFields ?? []) { + const column = hashColumnOf(field); + if (column) return make('filter', column); + } + for (const field of opts.sortFields ?? []) { + const column = hashColumnOf(field); + if (column) return make('sort', column); + } + return undefined; +} + +/** + * [#21207] The columns of a stored-metadata table a `search` never scans: the + * body column and the content-hash columns. A search is a substring filter + * evaluated server-side over every scanned column, so over these columns it is + * the same verifier a filter is — over the stored hash, and over the stored + * body (a withheld credential rebuilt by prefix probing) — and the engine's + * auto-default search set includes every one of them, since all three are + * text columns. + */ +export const STORED_METADATA_UNSEARCHABLE_COLUMNS: readonly string[] = Object.freeze([ + STORED_BODY_COLUMN, + ...STORED_METADATA_HASH_COLUMNS, +]); + +/** + * The refusal for an EXPLICIT search field list (`searchFields`, or the + * object-form `search.fields`) on a stored-metadata table that names a column + * of {@link STORED_METADATA_UNSEARCHABLE_COLUMNS}, or `undefined`. Same + * envelope as the evaluate refusals: `INVALID_FIELD` / 400. + */ +export function storedMetadataSearchRefusal( + object: string, + requested: readonly string[], + param: string, +): Error | undefined { + if (!isStoredMetadataBodyObject(object)) return undefined; + const column = requested.find((name) => STORED_METADATA_UNSEARCHABLE_COLUMNS.includes(name)); + if (column === undefined) return undefined; + const err: any = new Error( + `Cannot search '${object}' in '${column}' (${param}): the query was not run. A search evaluates ` + + `every column it scans row by row, and the '${column}' column holds ${column === STORED_BODY_COLUMN + ? 'a stored metadata body with credential material this door withholds' + : 'the stored content hash of a metadata body, which this door serves only in keyed form'}, so a ` + + `search over it would answer guesses about withheld values. Search ${USABLE_COLUMNS} instead.`, + ); + err.code = 'INVALID_FIELD'; + err.status = 400; + err.field = column; + err.fields = [column]; + err.object = object; + err.param = param; + return err; +} diff --git a/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts b/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts index fdcdcbf5510..92213f79819 100644 --- a/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts +++ b/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts @@ -458,7 +458,10 @@ describe('[#7748] the audit trail records the whole lifecycle, not only `save`', code: 'metadata_conflict', actor: 'admin', }); - expect(String(denial.note)).toContain('sha256:stale'); + // [#21207] The note names which side was present, never a version + // token or a stored hash: a copy carries no hash (fork three, ruling A). + expect(String(denial.note)).not.toContain('sha256:stale'); + expect(String(denial.note)).toContain('[withheld]'); }); // ── the read door ─────────────────────────────────────────────────────── diff --git a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts index 17ea0331728..6a6e155a971 100644 --- a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts +++ b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts @@ -45,7 +45,8 @@ const TEST_KEY = 'served-content-hash-test-key'; const keyedDigest = async (plain: string): Promise => `hmac-sha256:${createHmac('sha256', TEST_KEY).update(plain, 'utf8').digest('hex')}`; -const SHA256 = /sha256:[0-9a-f]{64}/; +/** An unkeyed content hash — the keyed form's `hmac-sha256:` prefix is not one. */ +const SHA256 = /(?= 400 && status < 500; } +/** + * [#21207] A caller's version token that names no current stored head, found by + * the protocol's own KEYED comparison before the repository is asked (see + * {@link ObjectStackProtocolImplementation.storedParentForToken}). + * + * A `ConflictError`, so each door's existing conflict branch — the 409 + * `METADATA_CONFLICT` and its decision-audit row — handles it unchanged; told + * apart from the repository's own race conflict because its `expectedParent` + * is the CALLER's token rather than a stored hash, and `unverifiable` marks the + * deployment with no crypto provider, where no token can be checked at all. + * The message is replaced: the base class prints both values, and one of them + * is the stored hash. + */ +class InboundVersionConflictError extends ConflictError { + constructor( + ref: { org: string; type: string; name: string }, + token: string, + currentStored: string | null, + readonly unverifiable: boolean, + ) { + super(ref as ConstructorParameters[0], token, currentStored); + this.message = `Conflict on ${ref.type}/${ref.name}: the version token sent ` + + (unverifiable ? 'cannot be checked (no crypto provider is registered)' : 'is not the current version'); + } +} + /** * [#8136] The client-facing sentence for a failed overlay delete: the caller's * own refusal when they declared one, and otherwise a stable line that names @@ -11056,6 +11092,62 @@ export class ObjectStackProtocolImplementation implements throw err; } + /** + * [#21207] A `search` on a stored-metadata table never scans its body or + * content-hash columns ({@link STORED_METADATA_UNSEARCHABLE_COLUMNS}). + * + * A search is a substring filter the engine evaluates over every column it + * scans, and with no `searchableFields` declared it scans every text-like + * column — the stored body and both stored hashes among them. Over those + * it is the verifier the filter refusals close: a guessed hash, or a guessed + * prefix of withheld credential material, returns the row exactly when it + * is right. So an explicit field list naming one is refused + * (`INVALID_FIELD` / 400, the evaluate refusals' envelope), and a search + * that names none is handed to the engine with the object's searchable set + * minus those columns — the engine intersects an override with that set and + * never widens it. Runs after {@link assertSearchFieldsAreSearchable}, so a + * name that is not searchable at all keeps its own answer. + */ + private narrowStoredMetadataSearch( + object: string, + options: Record, + wireSpelling: Record, + ): void { + if (!isStoredMetadataBodyObject(object)) return; + const objectForm = options.search !== null && typeof options.search === 'object'; + const [explicit, param] = options.searchFields != null + ? [options.searchFields, wireSpelling.searchFields ?? 'searchFields'] + : objectForm && options.search.fields != null + ? [options.search.fields, wireSpelling.search ?? 'search'] + : [undefined, '']; + const names: string[] = typeof explicit === 'string' + ? explicit.split(',').map((s: string) => s.trim()).filter(Boolean) + : Array.isArray(explicit) ? explicit.filter((f: unknown): f is string => typeof f === 'string') : []; + if (names.length > 0) { + const refusal = storedMetadataSearchRefusal(object, names, param); + if (refusal) throw refusal; + return; + } + if (options.search == null) return; + const gate = this.resolveQueryFields(object); + // No field map: the engine has none to expand a search over either. + if (!gate) return; + const { allowed } = resolveSearchFieldResolution({ + fields: gate.fields, + searchableFields: gate.schema?.searchableFields, + displayField: gate.schema?.nameField ?? gate.schema?.displayNameField, + }); + const narrowed = allowed.filter((field) => !STORED_METADATA_UNSEARCHABLE_COLUMNS.includes(field)); + if (narrowed.length === 0) { + // An empty override is ABSENT to the engine, which would then scan + // the whole default set — these columns included. Refuse instead. + const refusal = storedMetadataSearchRefusal(object, allowed, wireSpelling.search ?? 'search'); + if (refusal) throw refusal; + return; + } + options.searchFields = narrowed; + } + /** * [#4254] GROUP-BY axis. A grouping target the object does not have is * refused (`400 INVALID_FIELD`); a grouping target the spec cannot read is @@ -11584,6 +11676,9 @@ export class ObjectStackProtocolImplementation implements request.object, (options.search as any).fields, wireSpelling.search ?? 'search', ); } + // [#21207] …and on a stored-metadata table a search never scans the body + // or content-hash columns: refused when named, narrowed away otherwise. + this.narrowStoredMetadataSearch(request.object, options, wireSpelling); // Boolean fields for (const key of ['distinct', 'count']) { @@ -11709,13 +11804,22 @@ export class ObjectStackProtocolImplementation implements ? (options.aggregations as ReadonlyArray<{ filter?: unknown }>).flatMap((a) => collectFilterFieldKeys(a?.filter)) : []; - const bodyPredicateRefusal = storedMetadataBodyPredicateRefusal(request.object, { - filterFields: [...collectFilterFieldKeys(options.where), ...aggregationFilterFields], - sortFields: Array.isArray(options.orderBy) - ? (options.orderBy as ReadonlyArray<{ field?: unknown }>).map((e) => e?.field) - : [], - }); + const filterFields = [...collectFilterFieldKeys(options.where), ...aggregationFilterFields]; + const sortFields = Array.isArray(options.orderBy) + ? (options.orderBy as ReadonlyArray<{ field?: unknown }>).map((e) => e?.field) + : []; + const bodyPredicateRefusal = storedMetadataBodyPredicateRefusal(request.object, { filterFields, sortFields }); if (bodyPredicateRefusal) throw bodyPredicateRefusal; + // [#21207] The same three shapes on the stored CONTENT-HASH columns + // (maintainer ruling A on the second execution fork): a group key would + // serve the stored hash, a filter on it is an online verifier, a sort + // orders by it. Refused in the body column's envelope, before the engine. + const hashEvaluateRefusal = storedMetadataHashEvaluateRefusal(request.object, { + groupBy: options.groupBy, + filterFields, + sortFields, + }); + if (hashEvaluateRefusal) throw hashEvaluateRefusal; // Route to engine.aggregate() when the query has GROUP BY / aggregations. // engine.find() does not do in-memory aggregation fallback, so without @@ -11813,10 +11917,18 @@ export class ObjectStackProtocolImplementation implements // projection named only the body, and taken back off before serving. const bodyProjection = storedMetadataBodyProjection(request.object, options.fields); if (bodyProjection.addedType) options.fields = bodyProjection.fields; - const records = redactStoredMetadataRows( + // [#21207] …and its stored CONTENT HASH (`checksum`, `previous_checksum`) + // is served in keyed form — never the stored value, which beside the + // projected body confirms a guess at the withheld material offline — + // and omitted when no crypto provider is registered. + const records = await serveStoredMetadataHashColumnRows( request.object, - await this.engine.find(request.object, options), - { dropType: bodyProjection.addedType }, + redactStoredMetadataRows( + request.object, + await this.engine.find(request.object, options), + { dropType: bodyProjection.addedType }, + ), + this.storedHashDigest(), ); // Pagination metadata. When a `limit` is present the response is a single // page, so `records.length` is the page size — NOT the match total. Run a @@ -11941,7 +12053,13 @@ export class ObjectStackProtocolImplementation implements return { object: request.object, id: request.id, - record: redactStoredMetadataRow(request.object, result, { dropType: bodyProjection.addedType }), + // [#21207] Same served form as the list path: the content-hash + // columns keyed, or omitted with no crypto provider. + record: await serveStoredMetadataHashColumns( + request.object, + redactStoredMetadataRow(request.object, result, { dropType: bodyProjection.addedType }), + this.storedHashDigest(), + ), }; } throw recordNotFoundError(request.object, request.id); @@ -15579,6 +15697,103 @@ export class ObjectStackProtocolImplementation implements await this.recordMetadataAudit(ObjectStackProtocolImplementation.optimisticConflictAuditEntry(args)); } + // ----------------------------------------------------------------------- + // [#21207] The stored content hash, as the `/meta` doors serve and take it + // ----------------------------------------------------------------------- + // + // Maintainer ruling B on #21207: the stored content hash of a metadata body + // stays the canonical SHA-256 at rest — the repository contract, its + // producers and the parent links are untouched — but no door hands it out. + // It is a hash over the WHOLE stored body, withheld credential material + // included, so beside the projected body it confirms a guess at that + // material offline. Every door that serves it serves the crypto provider's + // keyed digest of it; every door that takes a version token back compares + // the token in that same form and hands the STORED value to the repository; + // with no provider registered nothing is served and every token is refused. + + /** + * The registered crypto provider's keyed digest, read from the engine at + * the moment of use — a host registers the provider AFTER the kernel starts, + * so a value read once at construction would answer "none" for good. + * `undefined` when none is registered (or the host engine has no such + * accessor, which is the same fact): the doors then withhold and refuse. + */ + private storedHashDigest(): StoredHashDigest | undefined { + const accessor = this.engine?.getKeyedDigest; + return typeof accessor === 'function' ? accessor.call(this.engine) : undefined; + } + + /** + * A write receipt's `version` — the version token a caller sends back as + * `If-Match` — for a write whose stored content hash is `stored`: its keyed + * digest. With no crypto provider the deployment issues no token, and the + * receipt's required `version` carries the empty string: a value that names + * nothing and that every inbound comparison refuses. + */ + private async receiptVersion(stored: string): Promise { + return (await servedContentHash(stored, this.storedHashDigest())) ?? ''; + } + + /** + * Resolve a caller's version token to the STORED head it names, for a write + * whose current stored head is `currentStored`: the token must equal the + * keyed digest of that head, and the stored value is what the repository's + * own optimistic lock then compares. `null` keeps its meaning ("expect no + * row") and carries no hash. Anything else — the raw stored hash a pre-keying + * client still holds, a stale token, a token sent to a deployment with no + * crypto provider — is an {@link InboundVersionConflictError}, which the + * door's conflict branch answers 409. + */ + private async storedParentForToken( + ref: { org: string; type: string; name: string }, + token: string | null, + currentStored: string | null, + ): Promise { + if (token === null) return null; + const digest = this.storedHashDigest(); + if (!digest) throw new InboundVersionConflictError(ref, token, currentStored, true); + if (currentStored !== null && (await digest(currentStored)) === token) return currentStored; + throw new InboundVersionConflictError(ref, token, currentStored, false); + } + + /** + * The 409 `METADATA_CONFLICT` a door answers for a `ConflictError`, whose + * text and attributes carry the SERVED (keyed) form of a stored hash or + * none at all — never a stored hash. `subject` names the item, `prefix` is + * the door's own first sentence. + * + * - a repository race (the stored head moved between the door's read and + * its write): `Expected parent X but current is Y`, both keyed; + * - a caller's token naming no current head: the keyed current head; + * - no crypto provider: no value, and the remedy. + * + * A side with no served form is `[withheld]`; an absent side is `null`. + */ + private async metadataConflictRefusal(err: ConflictError, subject: string, prefix: string): Promise { + const conflict: any = new Error(subject); + conflict.code = 'METADATA_CONFLICT'; + conflict.status = 409; + if (err instanceof InboundVersionConflictError && err.unverifiable) { + conflict.message = `${subject}: the version token sent cannot be checked, so the write was not run. ` + + 'This deployment registers no crypto provider, so it issues no version tokens and compares none. ' + + 'Send the write without a version token (no If-Match) to write unconditionally, or register a ' + + 'crypto provider (engine.setCryptoProvider) so version tokens are issued and checked.'; + return conflict; + } + const digest = this.storedHashDigest(); + const show = (served: string | null | undefined) => (served === undefined ? '[withheld]' : served ?? 'null'); + const current = await servedContentHash(err.actualHead, digest); + if (current !== undefined) conflict.actualHead = current; + if (err instanceof InboundVersionConflictError) { + conflict.message = `${prefix} The version token sent is not the current version (current is ${show(current)}).`; + return conflict; + } + const expected = await servedContentHash(err.expectedParent, digest); + if (expected !== undefined) conflict.expectedParent = expected; + conflict.message = `${prefix} Expected parent ${show(expected)} but current is ${show(current)}.`; + return conflict; + } + /** * [#8594] The same row as a VALUE, for the site that must not write it * where the conflict is caught — see {@link lockWriteRefusal} for the full @@ -15608,7 +15823,15 @@ export class ObjectStackProtocolImplementation implements ...(args.actor ? { actor: args.actor } : {}), source: args.source, ...(args.requestId ? { requestId: args.requestId } : {}), - note: `expected parent ${args.expectedParent ?? 'null'} but current is ${args.actualHead ?? 'null'}`, + // [#21207] Fork three, ruling A: a COPY never carries the stored + // content hash — not raw (an offline verifier, at rest and served to + // every audit reader), and not keyed either (a copy has no use for a + // version token, and a keyed value would die with the key). The + // note keeps its sentence and says which side was absent; a value + // is `[withheld]`. `os migrate audit-metadata-bodies` rewrites the + // notes written before this to exactly this text. + note: `expected parent ${args.expectedParent == null ? 'null' : '[withheld]'} ` + + `but current is ${args.actualHead == null ? 'null' : '[withheld]'}`, }; } @@ -16800,7 +17023,13 @@ export class ObjectStackProtocolImplementation implements } } - async saveMetaItem(request: { type: string, name: string, item?: any, organizationId?: string, parentVersion?: string | null, actor?: string, force?: boolean, mode?: 'draft' | 'publish', packageId?: string | null, source?: string, writeFace?: MetadataWriteFace }) { + // [#21207] `parentVersion` is a CALLER's version token — the keyed form a + // receipt served — and is compared in that form (`storedParentForToken`). + // `storedParentVersion` is the in-process twin for a caller that read the + // STORED content hash itself (`migrateStoredMetadata`): it reaches the + // repository as given. ⛔ No transport sets it — every door builds its + // request field by field from named inputs, never by spreading a body. + async saveMetaItem(request: { type: string, name: string, item?: any, organizationId?: string, parentVersion?: string | null, storedParentVersion?: string | null, actor?: string, force?: boolean, mode?: 'draft' | 'publish', packageId?: string | null, source?: string, writeFace?: MetadataWriteFace }) { // [commit fd6bdf89f] The ADR-0112 envelope this refusal always owed. Every OTHER // refusal in this method declares `code` AND `status` // (`NOT_OVERRIDABLE`/403, `NOT_CREATABLE`/403, `ITEM_LOCKED`/403, @@ -17672,8 +17901,9 @@ export class ObjectStackProtocolImplementation implements org: orgId ?? 'env', } as Parameters[0]; let parentVersion: string | null; - if (request.parentVersion !== undefined) { - parentVersion = request.parentVersion; + if (request.storedParentVersion !== undefined) { + // [#21207] An in-process caller that read the stored hash itself. + parentVersion = request.storedParentVersion; } else { // Parent is scoped to the lifecycle we're about to write: // a draft's parent is the current draft hash (or null @@ -17685,7 +17915,22 @@ export class ObjectStackProtocolImplementation implements state: mode === 'draft' ? 'draft' : 'active', packageId: request.packageId ?? null, }); - parentVersion = current?.hash ?? null; + const currentStored = current?.hash ?? null; + if (request.parentVersion === undefined) { + parentVersion = currentStored; + } else { + // [#21207] A caller's version token names the stored head only + // in keyed form; the repository's own lock then runs on the + // stored value. A token naming no current head — the raw + // stored hash included — is answered here, in this door's own + // conflict envelope and with its own audit row. + try { + parentVersion = await this.storedParentForToken(ref, request.parentVersion, currentStored); + } catch (err: unknown) { + if (err instanceof ConflictError) throw await this.saveConflict(err, request, orgId, writeSource); + throw err; + } + } } // [#8154] THE WRITE-PATH INVERSE of the read exits' credential // redaction — the half without which this card's fix is a DATA-LOSS @@ -17795,7 +18040,9 @@ export class ObjectStackProtocolImplementation implements }); return { success: true, - version: result.version, + // [#21207] The version token: the keyed form of the stored + // content hash, never the stored value (see `receiptVersion`). + version: await this.receiptVersion(result.version), seq: result.seq, ...(projectionApplied ? { projectionApplied } : {}), // [#4717] #4463 D3's advisory half, finally on the response. @@ -17855,31 +18102,42 @@ export class ObjectStackProtocolImplementation implements : `Saved ${singularTypeForRepo} '${request.name}' (env-wide, state=${mode === 'draft' ? 'draft' : 'active'}) [seq=${result.seq}]`), }; } catch (err: any) { - if (err instanceof ConflictError) { - const conflict = new Error( - `${request.type}/${request.name} has been modified since you loaded it. ` - + `Expected parent ${err.expectedParent ?? 'null'} but current is ${err.actualHead ?? 'null'}.`, - ); - (conflict as any).code = 'METADATA_CONFLICT'; - (conflict as any).status = 409; - (conflict as any).expectedParent = err.expectedParent; - (conflict as any).actualHead = err.actualHead; - await this.recordOptimisticConflictAudit({ - type: request.type, - name: request.name, - organizationId: orgId, - operation: 'save', - ...(request.actor ? { actor: request.actor } : {}), - source: writeSource, - expectedParent: err.expectedParent, - actualHead: err.actualHead, - }); - throw conflict; - } + if (err instanceof ConflictError) throw await this.saveConflict(err, request, orgId, writeSource); throw err; } } + /** + * The save door's 409 for a `ConflictError` — the repository's race, or a + * caller's token the keyed comparison refused — plus its decision-audit row. + * One builder for both, so the two cannot answer in different words. + * [#21207] The text and attributes carry keyed values or none + * ({@link metadataConflictRefusal}); the audit row carries no hash at all. + */ + private async saveConflict( + err: ConflictError, + request: { type: string; name: string; actor?: string }, + orgId: string | null, + writeSource: string, + ): Promise { + const conflict = await this.metadataConflictRefusal( + err, + `${request.type}/${request.name}`, + `${request.type}/${request.name} has been modified since you loaded it.`, + ); + await this.recordOptimisticConflictAudit({ + type: request.type, + name: request.name, + organizationId: orgId, + operation: 'save', + ...(request.actor ? { actor: request.actor } : {}), + source: writeSource, + expectedParent: err.expectedParent, + actualHead: err.actualHead, + }); + return conflict; + } + /** * `os migrate meta --stored` — canonicalize `sys_metadata` rows in place so * the read-path conversion chain has a finish line (#4327). @@ -18316,7 +18574,10 @@ export class ObjectStackProtocolImplementation implements name: base.name, item, mode: state === 'draft' ? 'draft' : 'publish', - parentVersion: row.checksum ?? null, + // [#21207] The STORED hash this pass read itself — the + // in-process spelling, never compared in keyed form (a + // deployment with no crypto provider still migrates). + storedParentVersion: row.checksum ?? null, packageId, force: true, source: 'migrate-stored', @@ -18432,7 +18693,18 @@ export class ObjectStackProtocolImplementation implements const opts: { sinceSeq?: number; limit?: number } = {}; if (request.sinceSeq !== undefined) opts.sinceSeq = request.sinceSeq; if (request.limit !== undefined) opts.limit = request.limit; - for await (const ev of repo.history(ref, opts)) events.push(ev); + // [#21207] Each event's hash and parent hash are served in keyed form — + // the same value the write receipts hand out, so an event still names + // the token a caller holds — and `null` with no crypto provider (a + // delete event's own `null` is kept either way). + const digest = this.storedHashDigest(); + for await (const ev of repo.history(ref, opts)) { + events.push({ + ...ev, + hash: (await servedContentHash(ev.hash, digest)) ?? null, + parentHash: (await servedContentHash(ev.parentHash, digest)) ?? null, + }); + } return { events }; } @@ -18704,7 +18976,8 @@ export class ObjectStackProtocolImplementation implements advisories?: RuntimeAuthoringIssue[]; } = { success: true, - version: result.version, + // [#21207] Keyed, never the stored content hash (`receiptVersion`). + version: await this.receiptVersion(result.version), seq: result.seq, message: `Published draft — type=${request.type}, name=${request.name} [seq=${result.seq}]`, // [#9176] Omitted-when-empty, never `advisories: []` — a clean @@ -18962,14 +19235,12 @@ export class ObjectStackProtocolImplementation implements return { singularType, orgId, advisories: runtimeAdvisories, result }; } catch (err: any) { if (err instanceof ConflictError) { - const conflict: any = new Error( - `${request.type}/${request.name} published row advanced while you held the draft. ` - + `Expected parent ${err.expectedParent ?? 'null'} but current is ${err.actualHead ?? 'null'}.`, + // [#21207] Keyed values or none in the text and attributes. + const conflict = await this.metadataConflictRefusal( + err, + `${request.type}/${request.name}`, + `${request.type}/${request.name} published row advanced while you held the draft.`, ); - conflict.code = 'METADATA_CONFLICT'; - conflict.status = 409; - conflict.expectedParent = err.expectedParent; - conflict.actualHead = err.actualHead; // [#8594] Attached, not written — same reason as the lock gate // above. The repository's own transaction has already unwound by // the time this `catch` runs, but the BATCH caller's has not. @@ -20232,7 +20503,10 @@ export class ObjectStackProtocolImplementation implements // element's bytes are unchanged, and absence means "nothing to // report", never "the gate did not run". published.push({ - type: p.d.type, name: p.d.name, version: p.version, + // [#21207] Each element's version token is keyed, like the + // single-item doors' (`receiptVersion`); `p.version` stays the + // stored hash for everything internal. + type: p.d.type, name: p.d.name, version: await this.receiptVersion(p.version), ...(p.advisories.length > 0 ? { advisories: p.advisories } : {}), }); try { @@ -22502,21 +22776,20 @@ export class ObjectStackProtocolImplementation implements }); return { success: true, - version: result.version, + // [#21207] Keyed, never the stored content hash (`receiptVersion`). + version: await this.receiptVersion(result.version), seq: result.seq, restoredFromVersion: request.toVersion, message: `Reverted to version ${request.toVersion} — type=${request.type}, name=${request.name} [seq=${result.seq}]`, }; } catch (err: any) { if (err instanceof ConflictError) { - const conflict: any = new Error( - `${request.type}/${request.name} advanced during rollback. ` - + `Expected parent ${err.expectedParent ?? 'null'} but current is ${err.actualHead ?? 'null'}.`, + // [#21207] Keyed values or none in the text and attributes. + const conflict = await this.metadataConflictRefusal( + err, + `${request.type}/${request.name}`, + `${request.type}/${request.name} advanced during rollback.`, ); - conflict.code = 'METADATA_CONFLICT'; - conflict.status = 409; - conflict.expectedParent = err.expectedParent; - conflict.actualHead = err.actualHead; await this.recordOptimisticConflictAudit({ type: request.type, name: request.name, @@ -23029,8 +23302,13 @@ export class ObjectStackProtocolImplementation implements // Last-write-wins parent resolution unless the caller pinned // an explicit version (Studio's "Reset" button is unpinned; // a future "delete vN" flow can pass parentVersion). - const parentVersion: string = request.parentVersion !== undefined - ? (request.parentVersion ?? current.hash) + // [#21207] A pinned version is a caller's token, compared in + // keyed form against the current stored head + // (`storedParentForToken`); the repository's lock then runs on + // the stored value. A token naming no current head throws a + // `ConflictError`, answered by the conflict branch below. + const parentVersion: string = typeof request.parentVersion === 'string' + ? ((await this.storedParentForToken(ref, request.parentVersion, current.hash)) ?? current.hash) : current.hash; const result = await repo.delete(ref, { @@ -23131,14 +23409,12 @@ export class ObjectStackProtocolImplementation implements }; } catch (err: any) { if (err instanceof ConflictError) { - const conflict = new Error( - `${request.type}/${request.name} has been modified since you loaded it. ` - + `Expected parent ${err.expectedParent ?? 'null'} but current is ${err.actualHead ?? 'null'}.`, + // [#21207] Keyed values or none in the text and attributes. + const conflict = await this.metadataConflictRefusal( + err, + `${request.type}/${request.name}`, + `${request.type}/${request.name} has been modified since you loaded it.`, ); - (conflict as any).code = 'METADATA_CONFLICT'; - (conflict as any).status = 409; - (conflict as any).expectedParent = err.expectedParent; - (conflict as any).actualHead = err.actualHead; await this.recordOptimisticConflictAudit({ type: request.type, name: request.name, diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 6b787347f64..c7f53cdf359 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -8560,6 +8560,26 @@ export class ObjectQL implements IObjectQLEngine { } } + /** + * [#21207] Read accessor for the registered provider's keyed digest + * (`ICryptoProvider.keyedDigest`), or `undefined` while no provider is + * registered. + * + * The ONE way a consumer outside this engine reaches the server-held key: + * the doors that serve a stored metadata content hash serve + * `keyedDigest(stored)` instead, and compare a caller's version token in that + * same form. Deliberately narrower than the provider itself — a consumer + * that needs a keyed digest gets that one primitive, never `decrypt`. + * + * Read at the moment of use, never cached by the caller: a host injects the + * provider AFTER the kernel starts (see {@link setCryptoProvider}), so a + * value captured at boot would still say "none" once one is registered. + */ + getKeyedDigest(): ((plain: string) => Promise) | undefined { + const provider = this.cryptoProvider; + return provider ? (plain: string) => provider.keyedDigest(plain) : undefined; + } + /** * [#8022] Observe crypto-provider registration. * From b56ab4c3ffc8bd6652a823ba5a8d89e94172e46c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 12:45:37 +0000 Subject: [PATCH 03/11] fix(mcp,plugin-audit,service-analytics,cli)!: the rest of the stored content-hash exit family (#21207) The MCP stdio reader serves the two hash columns keyed (or not at all) on query, get and the record resource, and refuses group, filter and sort on them. Analytics refuses them as members. The audit writer's copies drop them, and os migrate audit-metadata-bodies drops them from the copies already written and withholds the hashes in conflict notes and their copies. The family enumeration pin gains every member. Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .../commands/migrate/audit-metadata-bodies.ts | 32 ++- packages/mcp/src/plugin.ts | 23 +- packages/mcp/src/stdio-data-bridge.ts | 151 +++++++++++- .../stored-metadata-body-family.pin.test.ts | 66 ++++- .../plugins/plugin-audit/src/audit-writers.ts | 11 + .../src/stored-metadata-body-migration.ts | 230 +++++++++++++++--- .../src/stored-metadata-body-refusal.ts | 42 +++- 7 files changed, 492 insertions(+), 63 deletions(-) diff --git a/packages/cli/src/commands/migrate/audit-metadata-bodies.ts b/packages/cli/src/commands/migrate/audit-metadata-bodies.ts index f19931b6a7e..40b264029f4 100644 --- a/packages/cli/src/commands/migrate/audit-metadata-bodies.ts +++ b/packages/cli/src/commands/migrate/audit-metadata-bodies.ts @@ -44,16 +44,26 @@ async function confirm(question: string): Promise { * NEW writes; rows copied before the fix keep their cleartext. This command * rewrites them, projecting each copied body through the SAME redactor. * + * [#21207] The same copies also carried the copied row's stored CONTENT HASH + * (`checksum`, and the history row's `previous_checksum`) — a hash over the whole + * stored body, withheld credential material included — and the decision-audit + * note of a refused optimistic-lock write (`sys_metadata_audit`, and its ledger + * and activity copies) named both hashes. The writers no longer copy either; + * this command drops the hash columns from the copies already written and + * withholds the hashes in those notes, in the same pass. Operators run it once + * after upgrading, dry run first. + * * Dry run by default (writes nothing), `--apply` to rewrite. Idempotent: a - * second run finds nothing — a redacted copy has no credential left — so - * re-running and reading a clean report is the verification. No `sys_migration` - * flag is recorded: nothing gates irreversible behaviour on this rewrite (the - * posture `os migrate summary-nulls` takes). + * second run finds nothing — a redacted copy has no credential and no hash + * left — so re-running and reading a clean report is the verification. No + * `sys_migration` flag is recorded: nothing gates irreversible behaviour on this + * rewrite (the posture `os migrate summary-nulls` takes). */ export default class MigrateAuditMetadataBodies extends Command { static override description = - 'Rewrite at-rest cleartext metadata-body copies the audit writer left in sys_audit_log / sys_activity, ' + - 'projecting each copied body through the shared credential redactor. Dry run by default; --apply writes.'; + 'Rewrite the at-rest copies the audit writer left in sys_audit_log / sys_activity: project each copied ' + + 'metadata body through the shared credential redactor and drop its stored content hash; withhold the hashes ' + + 'a conflict note in sys_metadata_audit (and its copies) names. Dry run by default; --apply writes.'; static override examples = [ '$ os migrate audit-metadata-bodies', @@ -117,12 +127,12 @@ export default class MigrateAuditMetadataBodies extends Command { this.exit(1); return; } - printWarning('Apply mode rewrites audit/activity rows. Re-run with --yes to confirm, or run without --apply to preview.'); + printWarning('Apply mode rewrites audit/activity/decision rows. Re-run with --yes to confirm, or run without --apply to preview.'); this.exit(1); return; } const ok = await confirm( - chalk.bold('\nRewrite every audit/activity row carrying a stored metadata body on this database? [y/N] '), + chalk.bold('\nRewrite every audit/activity/decision row carrying a stored metadata body or content hash on this database? [y/N] '), ); if (!ok) { printInfo('Aborted — no changes made.'); @@ -188,14 +198,14 @@ export default class MigrateAuditMetadataBodies extends Command { printError(`${report.failures} row(s) could not be rewritten — re-run to finish them.`); } else if (apply && report.rewritten > 0) { printSuccess( - `Rewrote ${report.rewritten} audit/activity row(s). Re-run any time — it only revisits rows still carrying a body.`, + `Rewrote ${report.rewritten} audit/activity/decision row(s). Re-run any time — it only revisits rows still carrying a body or a hash.`, ); } else if (apply) { - printSuccess('Nothing to rewrite — no audit/activity row carries a stored metadata body.'); + printSuccess('Nothing to rewrite — no audit/activity/decision row carries a stored metadata body or content hash.'); } else if (report.rewritten > 0) { printInfo(`Dry run only — ${report.rewritten} row(s) would be rewritten. Re-run with --apply.`); } else { - printSuccess('Nothing to rewrite — no audit/activity row carries a stored metadata body.'); + printSuccess('Nothing to rewrite — no audit/activity/decision row carries a stored metadata body or content hash.'); } console.log(chalk.dim(` ${timer.display()}`)); console.log(''); diff --git a/packages/mcp/src/plugin.ts b/packages/mcp/src/plugin.ts index 0c099302d78..0092d820396 100644 --- a/packages/mcp/src/plugin.ts +++ b/packages/mcp/src/plugin.ts @@ -29,7 +29,9 @@ import { createStdioDataBridge, enforceApiExposure, GATED_ACTIONS, + serveStoredMetadataHashes, serveStoredMetadataRow, + type StoredHashDigest, } from './stdio-data-bridge.js'; import type { McpDataBridge } from './mcp-http-tools.js'; import { CONNECT_AGENT_UI_BUNDLE } from './connect-ui.js'; @@ -377,7 +379,14 @@ export class MCPServerPlugin implements Plugin { let dataBridge: McpDataBridge | undefined; if (shouldStart) { const apiKey = readEnvWithDeprecation('OS_MCP_STDIO_API_KEY', [], { silent: true }); - let ql: (IDataEngine & { find: (object: string, opts: unknown) => Promise }) | undefined; + let ql: + | (IDataEngine & { + find: (object: string, opts: unknown) => Promise; + // [#21207] The engine's keyed-digest accessor (objectql), probed + // per call: an engine without it serves no content hash. + getKeyedDigest?: () => StoredHashDigest | undefined; + }) + | undefined; try { ql = ctx.getService('objectql'); } catch { @@ -546,6 +555,10 @@ export class MCPServerPlugin implements Plugin { // wall that changed mid-session must take effect on the next call rather // than at the next process restart. See `resolveStdioTenancyPosture` for // why this is not hoisted next to the localization memo. + // [#21207] The crypto provider's keyed digest, read at each use — the + // host registers the provider after the kernel starts. + const storedHashDigest = (): StoredHashDigest | undefined => + typeof scopedQl.getKeyedDigest === 'function' ? scopedQl.getKeyedDigest() : undefined; const resolvePrincipal = async (): Promise => { const ec = await resolveStdioExecutionContext( scopedQl, @@ -561,6 +574,7 @@ export class MCPServerPlugin implements Plugin { engine: scopedQl, metadataService, resolvePrincipal, + keyedDigest: storedHashDigest, }); } else { // Functional degradation, said once and naming the remedy: two of the @@ -611,7 +625,12 @@ export class MCPServerPlugin implements Plugin { // `sys_metadata_history` row's body reaches this resource as its type's // read projection, never as the stored bytes — so the tool and the // resource cannot disagree about what a stored credential is. - return serveStoredMetadataRow(objectName, (row ?? null) as Record | null); + // [#21207] …and its stored content hash keyed, or not served at all. + return serveStoredMetadataHashes( + objectName, + serveStoredMetadataRow(objectName, (row ?? null) as Record | null), + storedHashDigest(), + ); }; ctx.logger.info( `[MCP] stdio transport principal-bound to OS_MCP_STDIO_API_KEY identity ${initial.userId} (RLS/FLS/tenant applied)`, diff --git a/packages/mcp/src/stdio-data-bridge.ts b/packages/mcp/src/stdio-data-bridge.ts index 24dd281d8d5..5d8c4aa1ea3 100644 --- a/packages/mcp/src/stdio-data-bridge.ts +++ b/packages/mcp/src/stdio-data-bridge.ts @@ -128,8 +128,19 @@ export interface StdioDataBridgeDeps { * live session, and a bridge built once at boot would outlive it. */ resolvePrincipal: () => Promise; + /** + * [#21207] The registered crypto provider's keyed digest + * (`ICryptoProvider.keyedDigest`, reached through the engine's + * `getKeyedDigest` accessor), read per call — a host registers the provider + * after the kernel starts. `undefined`, or a resolver answering `undefined`, + * means none: the stored content-hash columns are then not served at all. + */ + keyedDigest?: () => StoredHashDigest | undefined; } +/** The keyed-digest primitive the stored content-hash columns are served through. */ +export type StoredHashDigest = (plain: string) => Promise; + /** An object definition as `IMetadataService.getObject` hands it back. */ interface ObjectDef { name: string; @@ -422,6 +433,115 @@ export function storedMetadataBodyRefusal( return undefined; } +/** + * [#21207] The stored CONTENT-HASH columns of the same two tables: `checksum` + * (both) and the history table's `previous_checksum` (the parent's hash). + * + * Each is the canonical SHA-256 of the WHOLE stored body — withheld credential + * material included — kept as-is at rest. The engine returns it as stored, so + * this engine-only reader served it raw beside the projected body: an offline + * verifier for a guess at the withheld material, and an online one when + * filtered on. Per the maintainer's ruling on #21207 this transport serves the + * crypto provider's KEYED digest of it ({@link serveStoredMetadataHashes}) and + * refuses to evaluate it ({@link storedMetadataHashRefusal}) — the protocol + * data door's answer for the same columns + * (`@objectstack/metadata-protocol`'s `STORED_METADATA_HASH_COLUMNS`, which this + * package cannot import; `stored-metadata-body-family.pin.test.ts` pins the + * list to the object definitions). + */ +export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); + +/** The content-hash column a field reference reaches — the column, or a dotted path headed by it. */ +function hashColumnOf(field: unknown): string | undefined { + if (typeof field !== 'string') return undefined; + const head = field.split('.')[0] as string; + return STORED_METADATA_HASH_COLUMNS.includes(head) ? head : undefined; +} + +/** + * [#21207] The refusal for a read of a stored-metadata table whose call would + * EVALUATE a content-hash column — a grouping (whose keys would serve the + * stored values), a filter (a guessed hash matches its row) or a sort — or + * `undefined` when none is named. Same envelope as + * {@link storedMetadataBodyRefusal} (`INVALID_FIELD` / 400, naming the field, + * the object and the part), judged in the same order, and naming the columns + * that remain usable. + */ +export function storedMetadataHashRefusal( + object: string, + opts: { + where?: unknown; + orderBy?: ReadonlyArray<{ field?: unknown }>; + groupBy?: ReadonlyArray; + aggregations?: ReadonlyArray<{ field?: unknown; filter?: unknown }>; + }, +): McpStoredMetadataBodyRefusal | undefined { + if (!isStoredMetadataBodyObject(object)) return undefined; + const make = (param: 'groupBy' | 'filter' | 'sort', column: string): McpStoredMetadataBodyRefusal => { + const doing = param === 'groupBy' ? 'group' : param; + const err = new Error( + `Cannot ${doing} '${object}' by '${column}' (${param}): the query was not run. The '${column}' column holds ` + + `the stored content hash of a metadata body, computed over withheld credential material too, so this door ` + + `serves it only in keyed form; ${param === 'filter' + ? 'a filter on it compares a guess against the stored hash row by row, which confirms the guess' + : param === 'groupBy' ? 'a group key would serve the stored value itself' : 'a sort on it orders by the stored values'}. ` + + `Use '${STORED_METADATA_TYPE_COLUMN}', 'name', 'state' or another scalar column instead.`, + ) as McpStoredMetadataBodyRefusal; + err.code = 'INVALID_FIELD'; + err.status = 400; + err.field = column; + err.fields = [column]; + err.object = object; + err.param = param; + return err; + }; + for (const entry of opts.groupBy ?? []) { + const column = hashColumnOf( + entry && typeof entry === 'object' && !Array.isArray(entry) ? (entry as { field?: unknown }).field : entry, + ); + if (column) return make('groupBy', column); + } + const aggregations = Array.isArray(opts.aggregations) ? opts.aggregations : []; + for (const field of [...filterFieldKeys(opts.where), ...aggregations.flatMap((a) => filterFieldKeys(a?.filter))]) { + const column = hashColumnOf(field); + if (column) return make('filter', column); + } + for (const entry of Array.isArray(opts.orderBy) ? opts.orderBy : []) { + const column = hashColumnOf(entry?.field); + if (column) return make('sort', column); + } + return undefined; +} + +/** + * [#21207] Serve one row of `object` with its stored content-hash columns in + * their served form: each the keyed digest of the stored value, a `null` kept + * `null` — and both columns OMITTED when no crypto provider is registered. ⛔ + * Never the stored value. Rows of every other object, and rows carrying + * neither column, are returned by reference. A failing digest is not caught: + * the read fails rather than serving what it exists to replace. + * + * Exported within the package for the ADR-0101 record resource (`plugin.ts`), + * which serves through the same helpers as `bridge.get`. + */ +export async function serveStoredMetadataHashes( + object: string, + row: T, + digest: StoredHashDigest | undefined, +): Promise { + if (!isStoredMetadataBodyObject(object) || !row || typeof row !== 'object' || Array.isArray(row)) return row; + const record = row as Record; + if (!STORED_METADATA_HASH_COLUMNS.some((column) => column in record)) return row; + const out: Record = { ...record }; + for (const column of STORED_METADATA_HASH_COLUMNS) { + if (!(column in out)) continue; + const stored = out[column]; + if (!digest || (stored !== null && typeof stored !== 'string')) delete out[column]; + else out[column] = stored === null ? null : await digest(stored); + } + return out as T; +} + /** * The field projection to hand the engine for a read of `object`, given the * caller's own. @@ -486,7 +606,7 @@ export function serveStoredMetadataRows( * honoured rather than re-decided. */ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge { - const { engine, metadataService, resolvePrincipal } = deps; + const { engine, metadataService, resolvePrincipal, keyedDigest } = deps; const bridge: McpDataBridge = { async listObjects(): Promise { @@ -542,6 +662,9 @@ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge // on a stored metadata body is refused rather than evaluated. const bodyRefusal = storedMetadataBodyRefusal(object, { where: opts?.where, orderBy: opts?.orderBy }); if (bodyRefusal) throw bodyRefusal; + // [#21207] …and the same for a stored content-hash column. + const hashRefusal = storedMetadataHashRefusal(object, { where: opts?.where, orderBy: opts?.orderBy }); + if (hashRefusal) throw hashRefusal; const read = storedMetadataBodyReadFields(object, opts?.fields); const query: Record = {}; if (opts?.where) query.where = opts.where; @@ -549,10 +672,12 @@ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge if (opts?.orderBy) query.orderBy = opts.orderBy; if (typeof opts?.limit === 'number') query.limit = opts.limit; if (typeof opts?.offset === 'number') query.offset = opts.offset; - const records = serveStoredMetadataRows( - object, - unwrapRows(await engine.find(object, query, { context })), - { dropType: read.addedType }, + // [#21207] The content-hash columns are served keyed, or not at all. + const digest = keyedDigest?.(); + const records = await Promise.all( + serveStoredMetadataRows(object, unwrapRows(await engine.find(object, query, { context })), { + dropType: read.addedType, + }).map((row) => serveStoredMetadataHashes(object, row, digest)), ); return { object, records, total: records.length }; }, @@ -562,8 +687,13 @@ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge await enforceApiExposure(metadataService, object, GATED_ACTIONS.get, context); // `null` rather than a throw: `get_record` owns the not-found wording on // this path and already branches on a nullish record. - // [#21207] A stored metadata body is served as its type's projection. - return serveStoredMetadataRow(object, await findById(engine, object, id, context)); + // [#21207] A stored metadata body is served as its type's projection, + // and its stored content hash keyed (or not at all). + return serveStoredMetadataHashes( + object, + serveStoredMetadataRow(object, await findById(engine, object, id, context)), + keyedDigest?.(), + ); }, async create(object, data) { @@ -643,6 +773,13 @@ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge aggregations: opts?.aggregations, }); if (bodyRefusal) throw bodyRefusal; + // [#21207] …and a grouping or filter on a stored content-hash column. + const hashRefusal = storedMetadataHashRefusal(object, { + where: opts?.where, + groupBy: opts?.groupBy, + aggregations: opts?.aggregations, + }); + if (hashRefusal) throw hashRefusal; // No casts: `McpDataBridge.aggregate` declares the engine's own // `EngineAggregateOptions` slices since #8032, so the honest call // compiles — the two `as unknown as` casts this line used to carry diff --git a/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts b/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts index b9878da482a..7d7ee7644b7 100644 --- a/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts +++ b/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts @@ -38,6 +38,16 @@ * - the generic data door reads / groupBy / filter+sort → this package * (`protocol.data-door-stored-metadata-redaction.test.ts`, and the behaviour * asserted below). + * + * [#21207] Exit two — the same rows' stored CONTENT HASH (`checksum`, and the + * history table's `previous_checksum`), a hash over the whole stored body, + * withheld credential material included. Every surface that SERVES it serves + * the crypto provider's keyed digest (`keyed`), every surface that takes a + * version token back compares it keyed, every COPY drops it (`withheld`), and + * every EVALUATE shape refuses — enumerated as rows below, each with its pin, + * plus a third tooth: the hash columns the object definitions declare are + * exactly `STORED_METADATA_HASH_COLUMNS`, so a new hash-like column fails here + * instead of being served raw. */ import { describe, expect, it } from 'vitest'; @@ -49,8 +59,10 @@ import { } from '@objectstack/spec/kernel'; import { redactStoredMetadataRow, + STORED_METADATA_HASH_COLUMNS, storedMetadataBodyGroupingRefusal, storedMetadataBodyPredicateRefusal, + storedMetadataHashEvaluateRefusal, } from './metadata-redaction.js'; /** The generic data-door verbs this family's seam covers, and how. */ @@ -77,6 +89,28 @@ const FAMILY_SURFACES = [ disposition: 'refuses', pin: '@objectstack/mcp', }, + // [#21207] Exit two: the stored content hash of the same rows. + { + surface: '/meta save, publish, batch-publish and rollback receipts: the version token', + disposition: 'keyed', + pin: 'this package (protocol.served-content-hash.test.ts)', + }, + { surface: '/meta history read: each event\'s hash and parent hash', disposition: 'keyed', pin: 'this package (protocol.served-content-hash.test.ts)' }, + { surface: '/meta save and reset doors: the inbound version token', disposition: 'keyed', pin: 'this package (protocol.served-content-hash.test.ts)' }, + { surface: '409 conflict refusal: its text and attributes', disposition: 'keyed', pin: 'this package (protocol.served-content-hash.test.ts)' }, + { surface: 'decision-audit note of a conflict refusal', disposition: 'withheld', pin: 'this package (protocol.served-content-hash.test.ts)' }, + { surface: 'data door get / list: the content-hash columns', disposition: 'keyed', pin: 'this package (protocol.data-door-stored-content-hash.test.ts)' }, + { surface: 'data door: group / filter / sort on a content-hash column', disposition: 'refuses', pin: 'this package (protocol.data-door-stored-content-hash.test.ts)' }, + { surface: 'data door: search over the body column or a content-hash column', disposition: 'refuses', pin: 'this package (protocol.data-door-stored-content-hash.test.ts)' }, + { surface: 'MCP stdio engine-only reader: the content-hash columns on query / get / the record resource', disposition: 'keyed', pin: '@objectstack/mcp' }, + { surface: 'MCP stdio engine-only reader: group / filter / sort on a content-hash column', disposition: 'refuses', pin: '@objectstack/mcp' }, + { surface: 'audit / activity copy at write time: the content-hash columns', disposition: 'withheld', pin: '@objectstack/plugin-audit' }, + { surface: 'analytics members on a content-hash column', disposition: 'refuses', pin: '@objectstack/service-analytics' }, + { + surface: 'copies at rest (ledger snapshot and diff, activity copy, decision-audit note): os migrate audit-metadata-bodies', + disposition: 'withheld', + pin: '@objectstack/plugin-audit', + }, ] as const; /** A stored datasource body as it sits in the `metadata` column: serialized JSON with credential material. */ @@ -139,14 +173,15 @@ describe('[#21120] stored-metadata-body family — the data door exposes only co } }); - it('enumerates every family surface with a disposition (seam | refuses) and an owning pin', () => { + it('enumerates every family surface with a disposition (seam | keyed | withheld | refuses) and an owning pin', () => { for (const s of FAMILY_SURFACES) { - expect(['seam', 'refuses']).toContain(s.disposition); + expect(['seam', 'keyed', 'withheld', 'refuses']).toContain(s.disposition); expect(s.pin.length).toBeGreaterThan(0); } // One row per the three local data-door surfaces + five cross-package ones - // (audit, analytics, realtime, and the MCP stdio reader's seam and refusal). - expect(FAMILY_SURFACES).toHaveLength(8); + // (audit, analytics, realtime, and the MCP stdio reader's seam and refusal), + // + [#21207] the thirteen surfaces of the stored content hash. + expect(FAMILY_SURFACES).toHaveLength(21); }); }); @@ -187,3 +222,26 @@ describe('[#21120] stored-metadata-body family — the local surfaces behave', ( expect(storedMetadataBodyPredicateRefusal('showcase_task', { filterFields: ['metadata'] })).toBeUndefined(); }); }); + +describe('[#21207] stored-metadata-body family — the stored content-hash columns', () => { + it('the hash columns the two object definitions declare are exactly STORED_METADATA_HASH_COLUMNS', () => { + const declared = (def: any) => Object.keys(def.fields ?? {}).filter((f) => /checksum|hash/i.test(f)).sort(); + expect(declared(SysMetadataObject)).toEqual(['checksum']); + expect(declared(SysMetadataHistoryObject)).toEqual(['checksum', 'previous_checksum']); + expect([...STORED_METADATA_HASH_COLUMNS].sort()).toEqual(['checksum', 'previous_checksum']); + }); + + it('data-door group / filter / sort on a hash column refuse (INVALID_FIELD / 400); a scalar column does not', () => { + for (const [object, column] of [['sys_metadata', 'checksum'], ['sys_metadata_history', 'previous_checksum']]) { + for (const opts of [{ groupBy: [column] }, { filterFields: [column] }, { sortFields: [column] }]) { + const err = storedMetadataHashEvaluateRefusal(object!, opts) as any; + expect(err?.code).toBe('INVALID_FIELD'); + expect(err?.status).toBe(400); + expect(err?.field).toBe(column); + } + } + expect(storedMetadataHashEvaluateRefusal('sys_metadata', { groupBy: ['type'], filterFields: ['name'], sortFields: ['state'] })) + .toBeUndefined(); + expect(storedMetadataHashEvaluateRefusal('file_blob', { filterFields: ['checksum'] })).toBeUndefined(); + }); +}); diff --git a/packages/plugins/plugin-audit/src/audit-writers.ts b/packages/plugins/plugin-audit/src/audit-writers.ts index 437819f06fb..82642c4db9d 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.ts @@ -43,6 +43,9 @@ import { STORED_METADATA_BODY_COLUMN, STORED_METADATA_TYPE_COLUMN, } from '@objectstack/spec/kernel'; +// [#21207] The same rows' stored content-hash columns — defined beside the +// at-rest rewrite that withholds the copies already written. +import { STORED_METADATA_HASH_COLUMNS } from './stored-metadata-body-migration.js'; // [commit 1408fe385 / #10101] The platform-row organization resolver, imported rather // than owned. It started life in THIS file (commit 1408fe385, honouring #8287's ruling) // and was promoted to `@objectstack/metadata-core` by the maintainer ruling @@ -1422,6 +1425,14 @@ export function installAuditWriters( if (outcome.ok) out[STORED_METADATA_BODY_COLUMN] = outcome.body; else delete out[STORED_METADATA_BODY_COLUMN]; } + // [#21207] …and the same row's stored CONTENT HASH is not copied at all + // (fork three, ruling A). It is a hash over the whole stored body, withheld + // credential material included, so a copy of it beside the projected body + // is an offline verifier, at rest and served to every ledger reader. A copy + // has no use for a version token, and the history table stays the lineage. + if (isStoredMetadataBodyObject(objectName)) { + for (const column of STORED_METADATA_HASH_COLUMNS) delete out[column]; + } if (dropComputed) { const defs = getFieldDefs(objectName); for (const key of Object.keys(out)) { diff --git a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts index c5350524e86..e6dfda2ac22 100644 --- a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts +++ b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts @@ -40,6 +40,23 @@ * left to withhold — so re-running and reading a clean report is the check, the * same posture `os migrate summary-nulls` takes. No `sys_migration` flag is * recorded: nothing gates irreversible behaviour on this rewrite. + * + * ## [#21207] The stored content hash of the same rows + * + * The same copies also carried the copied row's stored CONTENT HASH + * (`checksum`, and the history row's `previous_checksum`) — a hash over the + * whole stored body, withheld credential material included, i.e. an offline + * verifier for a guess at that material — and the protocol wrote both hashes + * of a refused optimistic-lock write into the decision-audit note + * (`sys_metadata_audit.note`, `code = 'metadata_conflict'`), which the writer + * then copied into the ledger and the activity feed again. The writers no + * longer do (fork three of the maintainer's ruling on #21207: a copy never + * carries the hash; the history table stays the lineage), and this rewrite now + * also drops the two columns from the copies already written + * ({@link withoutStoredHashColumns}) and withholds the hashes in the notes and + * their copies ({@link planDecisionNotePatch}) — the one operator-run rewrite + * this family already has, extended to a derived column of the same rows, never + * a second rewrite path. */ import { @@ -53,6 +70,148 @@ import type { IDataEngine } from '@objectstack/spec/contracts'; /** The two audit-family tables this writer copies stored metadata bodies into. */ export const STORED_METADATA_BODY_AUDIT_OBJECTS = ['sys_audit_log', 'sys_activity'] as const; +/** + * [#21207] The stored CONTENT-HASH columns of a stored-metadata-body row: + * `checksum` (both tables) and the history table's `previous_checksum`. Never + * copied by the audit writer (`audit-writers.ts`), dropped from the copies + * already written by this rewrite. The same list as + * `@objectstack/metadata-protocol`'s `STORED_METADATA_HASH_COLUMNS`, which this + * plugin cannot import; `stored-metadata-body-family.pin.test.ts` pins it to the + * object definitions. + */ +export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); + +/** [#21207] The decision-audit table whose conflict notes named both stored hashes. */ +export const METADATA_DECISION_AUDIT_OBJECT = 'sys_metadata_audit'; + +/** The persisted code of an optimistic-concurrency refusal's decision-audit row. */ +const CONFLICT_NOTE_CODE = 'metadata_conflict'; + +/** The conflict note's one sentence, as every writer of it has spelled it. */ +const CONFLICT_NOTE = /^expected parent (.+) but current is (.+)$/; + +/** + * Drop the stored content-hash columns from one parsed ledger snapshot (or one + * half of an activity pair). Unchanged when it carries neither. + */ +export function withoutStoredHashColumns(snapshot: unknown): { changed: boolean; value: unknown } { + if (!snapshot || typeof snapshot !== 'object' || Array.isArray(snapshot)) { + return { changed: false, value: snapshot }; + } + const record = snapshot as Record; + if (!STORED_METADATA_HASH_COLUMNS.some((column) => column in record)) return { changed: false, value: snapshot }; + const out: Record = { ...record }; + for (const column of STORED_METADATA_HASH_COLUMNS) delete out[column]; + return { changed: true, value: out }; +} + +/** + * The conflict note with each side's value withheld — `null` kept, anything + * else `[withheld]` — exactly the sentence the protocol writes now; `undefined` + * for a note that is not that sentence or already withholds both sides. + */ +function withheldConflictNote(note: unknown): string | undefined { + if (typeof note !== 'string') return undefined; + const match = CONFLICT_NOTE.exec(note); + if (!match) return undefined; + const side = (value: string) => (value === 'null' ? 'null' : '[withheld]'); + const rewritten = `expected parent ${side(match[1] as string)} but current is ${side(match[2] as string)}`; + return rewritten === note ? undefined : rewritten; +} + +/** A decision-audit row (or a ledger snapshot of one) that is a conflict refusal: its code says so, or it carries none. */ +function isConflictDecision(record: Record): boolean { + return record.code === undefined || record.code === CONFLICT_NOTE_CODE; +} + +/** Withhold the hashes in the note of one parsed snapshot of a decision-audit row. */ +function withheldNoteSnapshot(snapshot: unknown): { changed: boolean; value: unknown } { + if (!snapshot || typeof snapshot !== 'object' || Array.isArray(snapshot)) return { changed: false, value: snapshot }; + const record = snapshot as Record; + if (!isConflictDecision(record)) return { changed: false, value: snapshot }; + const note = withheldConflictNote(record.note); + return note === undefined ? { changed: false, value: snapshot } : { changed: true, value: { ...record, note } }; +} + +/** Parse → apply `step` → re-serialize one stored JSON string column. A non-string / unparseable value is left as-is. */ +function rewriteSerialized( + serialized: unknown, + step: (parsed: unknown) => { changed: boolean; value: unknown }, +): { changed: boolean; value: unknown } { + if (typeof serialized !== 'string' || serialized === '') return { changed: false, value: serialized }; + let parsed: unknown; + try { + parsed = JSON.parse(serialized); + } catch { + return { changed: false, value: serialized }; + } + const { changed, value } = step(parsed); + return changed ? { changed: true, value: JSON.stringify(value) } : { changed: false, value: serialized }; +} + +/** + * [#21207] The patch that withholds the stored hashes a conflict's + * decision-audit note named, for one row of `table`, or `null` when there is + * nothing to withhold: + * + * - `sys_metadata_audit` — the note itself (`code = 'metadata_conflict'`); + * - `sys_audit_log` / `sys_activity` — the ledger snapshot / activity pair the + * writer copied a `sys_metadata_audit` row into. + * + * Each value becomes `[withheld]` and a `null` side stays `null` — the sentence + * the protocol writes since this card, so a rewritten note and a new one read + * the same. Idempotent. + */ +export function planDecisionNotePatch( + table: string, + row: Record, +): Record | null { + if (table === METADATA_DECISION_AUDIT_OBJECT) { + if (!isConflictDecision(row)) return null; + const note = withheldConflictNote(row.note); + return note === undefined ? null : { note }; + } + if (row.object_name !== METADATA_DECISION_AUDIT_OBJECT) return null; + if (table === 'sys_audit_log') { + const patch: Record = {}; + for (const column of ['new_value', 'old_value']) { + const out = rewriteSerialized(row[column], withheldNoteSnapshot); + if (out.changed) patch[column] = out.value; + } + return Object.keys(patch).length > 0 ? patch : null; + } + if (table === 'sys_activity') { + const out = rewriteSerialized(row.metadata, (parsed) => { + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return { changed: false, value: parsed }; + const pair = parsed as Record; + const oldR = withheldNoteSnapshot(pair.old); + const newR = withheldNoteSnapshot(pair.new); + if (!oldR.changed && !newR.changed) return { changed: false, value: parsed }; + return { changed: true, value: { ...pair, old: oldR.value, new: newR.value } }; + }); + return out.changed ? { metadata: out.value } : null; + } + return null; +} + +/** Whether a copy row is about a stored-metadata-body row — its `object_name` says so, or it names none. */ +function copiesStoredMetadataRow(row: Record): boolean { + return row.object_name === undefined || row.object_name === null + || (STORED_METADATA_BODY_OBJECTS as ReadonlySet).has(String(row.object_name)); +} + +/** Redact the body AND drop the content-hash columns of one parsed snapshot of a stored-metadata row. */ +function redactStoredMetadataSnapshot( + snapshot: unknown, + typeHint: string | undefined, + dropHashes: boolean, +): { changed: boolean; value: unknown } { + const body = redactLedgerSnapshotBody(snapshot, typeHint); + if (!dropHashes) return body; + const hashes = withoutStoredHashColumns(body.value); + return { changed: body.changed || hashes.changed, value: hashes.value }; +} + /** The rewrite reads and writes as the platform, never as a user. */ const SYSTEM_CTX = { isSystem: true } as const; @@ -87,43 +246,42 @@ export function redactLedgerSnapshotBody( return { changed: true, value: rest }; } -/** Parse → redact → re-serialize one stored JSON string column. A non-string / unparseable value is left as-is. */ +/** + * Parse → redact (and, for a stored-metadata row's copy, drop the content-hash + * columns) → re-serialize one stored JSON string column. A non-string / + * unparseable value is left as-is. + */ function redactSerializedSnapshot( serialized: unknown, - typeHint?: string, + typeHint: string | undefined, + dropHashes: boolean, ): { changed: boolean; value: unknown } { - if (typeof serialized !== 'string' || serialized === '') return { changed: false, value: serialized }; - let parsed: unknown; - try { - parsed = JSON.parse(serialized); - } catch { - return { changed: false, value: serialized }; - } - const { changed, value } = redactLedgerSnapshotBody(parsed, typeHint); - if (!changed) return { changed: false, value: serialized }; - return { changed: true, value: JSON.stringify(value) }; + return rewriteSerialized(serialized, (parsed) => redactStoredMetadataSnapshot(parsed, typeHint, dropHashes)); } /** * The patch for one `sys_audit_log` row about a stored-metadata-body object, or - * `null` when the row holds no credential body to rewrite. + * `null` when the row holds no credential body to rewrite — and [#21207] no + * stored content hash to drop. */ export function planAuditRowPatch( row: Record, typeHint?: string, ): { new_value?: unknown; old_value?: unknown } | null { + const dropHashes = copiesStoredMetadataRow(row); const patch: { new_value?: unknown; old_value?: unknown } = {}; - const nv = redactSerializedSnapshot(row.new_value, typeHint); + const nv = redactSerializedSnapshot(row.new_value, typeHint, dropHashes); if (nv.changed) patch.new_value = nv.value; - const ov = redactSerializedSnapshot(row.old_value, typeHint); + const ov = redactSerializedSnapshot(row.old_value, typeHint, dropHashes); if (ov.changed) patch.old_value = ov.value; return nv.changed || ov.changed ? patch : null; } /** * The patch for one `sys_activity` row about a stored-metadata-body object, or - * `null` when it holds no credential body. `sys_activity.metadata` is the - * serialized pair `{ old, new }`, each half a ledger snapshot. + * `null` when it holds no credential body — and [#21207] no stored content hash + * to drop. `sys_activity.metadata` is the serialized pair `{ old, new }`, each + * half a ledger snapshot. */ export function planActivityRowPatch( row: Record, @@ -139,8 +297,9 @@ export function planActivityRowPatch( } if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; const pair = parsed as Record; - const oldR = redactLedgerSnapshotBody(pair.old, typeHint); - const newR = redactLedgerSnapshotBody(pair.new, typeHint); + const dropHashes = copiesStoredMetadataRow(row); + const oldR = redactStoredMetadataSnapshot(pair.old, typeHint, dropHashes); + const newR = redactStoredMetadataSnapshot(pair.new, typeHint, dropHashes); if (!oldR.changed && !newR.changed) return null; return { metadata: JSON.stringify({ ...pair, old: oldR.value, new: newR.value }) }; } @@ -182,8 +341,10 @@ function asArray(result: unknown): Record[] { /** * Rewrite every `sys_audit_log` / `sys_activity` row that copied a stored - * metadata body in cleartext, projecting the body through the shared redactor. - * Dry-run by default; `apply` writes. Idempotent. + * metadata body in cleartext, projecting the body through the shared redactor + * — and [#21207] every copy that carries a stored content hash (dropped) and + * every conflict note in `sys_metadata_audit` and its copies that names one + * (withheld). Dry-run by default; `apply` writes. Idempotent. */ export async function migrateStoredMetadataBodyCopies( engine: StoredMetadataBodyMigrationEngine, @@ -218,15 +379,18 @@ export async function migrateStoredMetadataBodyCopies( return type; }; - for (const object of STORED_METADATA_BODY_AUDIT_OBJECTS) { + // [#21207] The copies of a decision-audit row are read too (its conflict + // note named both stored hashes), and so is the decision-audit table itself. + const copiedObjects = [...STORED_METADATA_BODY_OBJECTS, METADATA_DECISION_AUDIT_OBJECT]; + const tables: ReadonlyArray<{ object: string; where: Record }> = [ + ...STORED_METADATA_BODY_AUDIT_OBJECTS.map((object) => ({ object, where: { object_name: { $in: copiedObjects } } })), + { object: METADATA_DECISION_AUDIT_OBJECT, where: { code: CONFLICT_NOTE_CODE } }, + ]; + for (const { object, where } of tables) { report.byObject[object] = { scanned: 0, rewritten: 0 }; let rows: Record[]; try { - const result = await engine.find( - object, - { where: { object_name: { $in: [...STORED_METADATA_BODY_OBJECTS] } } }, - { context: SYSTEM_CTX }, - ); + const result = await engine.find(object, { where }, { context: SYSTEM_CTX }); rows = asArray(result); } catch (e) { // A table this run could not read is NOT a clean table: counted as a @@ -243,9 +407,13 @@ export async function migrateStoredMetadataBodyCopies( for (const row of rows) { report.scanned += 1; report.byObject[object].scanned += 1; - const typeHint = await resolveType(row.record_id); - const patch = - object === 'sys_audit_log' ? planAuditRowPatch(row, typeHint) : planActivityRowPatch(row, typeHint); + // [#21207] A decision-audit row, or a copy of one: its conflict note. + const isDecision = object === METADATA_DECISION_AUDIT_OBJECT || row.object_name === METADATA_DECISION_AUDIT_OBJECT; + const patch = isDecision + ? planDecisionNotePatch(object, row) + : object === 'sys_audit_log' + ? planAuditRowPatch(row, await resolveType(row.record_id)) + : planActivityRowPatch(row, await resolveType(row.record_id)); if (!patch) continue; report.rewritten += 1; report.byObject[object].rewritten += 1; @@ -272,7 +440,7 @@ export async function migrateStoredMetadataBodyCopies( logger.info?.( `[stored-metadata-body-migration] ${apply ? 'rewrote' : 'would rewrite'} ${report.rewritten} of ${report.scanned} ` + - `audit/activity row(s) carrying a stored metadata body` + + `audit/activity/decision row(s) carrying a stored metadata body or a stored content hash` + (report.failures > 0 ? ` (${report.failures} failed — re-run to finish)` : ''), ); return report; diff --git a/packages/services/service-analytics/src/stored-metadata-body-refusal.ts b/packages/services/service-analytics/src/stored-metadata-body-refusal.ts index 1b27826be3e..6b4149c4674 100644 --- a/packages/services/service-analytics/src/stored-metadata-body-refusal.ts +++ b/packages/services/service-analytics/src/stored-metadata-body-refusal.ts @@ -34,10 +34,27 @@ * Every OTHER member of these objects — `type`, `name`, `scope`, `state`, * timestamps — is grouped, filtered and counted as before, so the Setup grids * and "All Metadata" dashboards that chart metadata by type keep working. Only - * the body column is refused. + * the body column, and since #21207 the two stored content-hash columns + * ({@link STORED_METADATA_HASH_COLUMNS}), are refused. */ import { isStoredMetadataBodyObject, STORED_METADATA_BODY_COLUMN } from '@objectstack/spec/kernel'; + +/** + * [#21207] The stored CONTENT-HASH columns of the same two tables: `checksum` + * (both) and the history table's `previous_checksum` (the parent's hash). + * + * Each is the canonical SHA-256 of the WHOLE stored body, withheld credential + * material included. Grouped by, it serves the stored values — beside the + * projected body every other door serves, an offline verifier for a guess at + * the withheld material; filtered on, it is an online one. Per the + * maintainer's ruling on #21207 they are refused here in the body column's + * envelope, in either role. The same list as + * `@objectstack/metadata-protocol`'s `STORED_METADATA_HASH_COLUMNS`, which this + * service cannot import; `stored-metadata-body-family.pin.test.ts` pins it to + * the object definitions. + */ +export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); import type { StandardErrorCode } from '@objectstack/spec/api'; import type { NamedField, NamedRead } from './field-read-admission.js'; @@ -57,18 +74,27 @@ export function storedMetadataBodyAnalyticsRefusal( // it outright (`field-read-admission.ts`, the expression refusal) — one // rule for expressions, never a second one in this module. if ('expression' in f) continue; - if (!isStoredMetadataBodyObject(f.object) || f.field !== STORED_METADATA_BODY_COLUMN) continue; + if (!isStoredMetadataBodyObject(f.object)) continue; + const isBody = f.field === STORED_METADATA_BODY_COLUMN; + if (!isBody && !STORED_METADATA_HASH_COLUMNS.includes(f.field)) continue; const param = f.role === 'aggregate' ? 'dimensions' : 'where'; const err = new Error( - `Cannot query '${f.object}' by '${STORED_METADATA_BODY_COLUMN}': the query was not run. The ` - + `${STORED_METADATA_BODY_COLUMN} column holds a stored metadata body, with stored credential material ` - + `withheld on every read exit; grouping, aggregating, filtering or sorting by it would evaluate the ` - + `stored body (a group key that cannot be projected, or a filter oracle that rebuilds a withheld value ` - + `by probing). Group, filter or sort by 'type', 'name' or another scalar column instead.`, + isBody + ? `Cannot query '${f.object}' by '${STORED_METADATA_BODY_COLUMN}': the query was not run. The ` + + `${STORED_METADATA_BODY_COLUMN} column holds a stored metadata body, with stored credential material ` + + `withheld on every read exit; grouping, aggregating, filtering or sorting by it would evaluate the ` + + `stored body (a group key that cannot be projected, or a filter oracle that rebuilds a withheld value ` + + `by probing). Group, filter or sort by 'type', 'name' or another scalar column instead.` + // [#21207] A stored content-hash column, in the same envelope. + : `Cannot query '${f.object}' by '${f.field}': the query was not run. The ${f.field} column holds the ` + + `stored content hash of a metadata body, computed over withheld credential material too, so every read ` + + `exit serves it only in keyed form; grouping, aggregating, filtering or sorting by it would evaluate the ` + + `stored hash (a group key that serves it, or a filter that confirms a guessed hash). Group, filter or ` + + `sort by 'type', 'name' or another scalar column instead.`, ) as Error & { code: string; status: number; field: string; object: string; param: string }; err.code = INVALID_FIELD; err.status = 400; - err.field = STORED_METADATA_BODY_COLUMN; + err.field = f.field; err.object = f.object; err.param = param; return err; From b844c8c2b41b56e8fbc55e07beaf60f3bdd48a17 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 13:24:20 +0000 Subject: [PATCH 04/11] fix(metadata-protocol,plugin-audit): withheld-marker spelling, superseded pins, changeset (#21207) The withheld marker reads (withheld) so no refusal opens with a bracketed tag; the two pins that asserted a conflict note carries a hash, and the batch publish conformance pin that asserted a token with no provider registered, follow the new contract. The changeset states the three caller and operator consequences. Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .changeset/21207-keyed-served-content-hash.md | 24 +++++++++++++++++++ .../src/protocol.lifecycle-audit-rows.test.ts | 2 +- ...rotocol.package-publish-audit-rows.test.ts | 5 +++- packages/metadata-protocol/src/protocol.ts | 10 ++++---- ...ackage-drafts-response-conformance.test.ts | 12 ++++++++-- .../src/stored-metadata-body-migration.ts | 6 ++--- .../stored-metadata-hash-migration.test.ts | 4 ++-- packages/rest/src/rest-server.ts | 11 +++++---- 8 files changed, 56 insertions(+), 18 deletions(-) create mode 100644 .changeset/21207-keyed-served-content-hash.md diff --git a/.changeset/21207-keyed-served-content-hash.md b/.changeset/21207-keyed-served-content-hash.md new file mode 100644 index 00000000000..1407cdc61cb --- /dev/null +++ b/.changeset/21207-keyed-served-content-hash.md @@ -0,0 +1,24 @@ +--- +'@objectstack/metadata-protocol': minor +'@objectstack/objectql': minor +'@objectstack/mcp': minor +'@objectstack/plugin-audit': minor +'@objectstack/service-analytics': minor +'@objectstack/cli': minor +--- + +fix(metadata-protocol)!: a metadata body's stored content hash is served and compared only in keyed form, never copied, and never evaluated (#21207) + +Clause-②: yes (narrowing) + + + +**BREAKING**: this narrows what the metadata doors serve and accept for the stored content hash of a metadata body — a hash over the whole stored body, withheld credential material included. Served beside the projected body it let a reader confirm a guess at that material offline; filtered on, it confirmed one online. It ships as `minor` under the launch-window convention for accept-set narrowings. + +**Three things change for callers and operators.** + +1. **A held version token gets one `409 METADATA_CONFLICT`.** Every door that hands out a metadata version token — the save, publish, package-publish and rollback receipts and the history read — now hands out the crypto provider's keyed digest of the stored hash instead of the hash itself, and the save and reset doors compare a token they are sent in that same form. A token a client held from before the upgrade is refused once; take the token from the next read or receipt and retry. A deployment that registers no crypto provider issues no token — the receipts' version is empty and the history read serves none — and refuses a sent token with the same `409`; send the write without one, or register a provider. +2. **Filter, sort and group on the two stored content-hash columns now answer `400 INVALID_FIELD`** — on the generic data door, the MCP stdio reader and the analytics door, before the engine runs. A data-door search over the two stored-metadata tables no longer scans those columns or the stored body column, and an explicit search-field list naming one answers the same `400`. Every other column of the two tables is served, filtered, sorted and grouped as before, and every other object is unchanged. +3. **Operators run `os migrate audit-metadata-bodies` once after upgrading, dry run first.** The audit ledger, the activity feed and the metadata decision-audit trail no longer copy the stored hash. The extended command drops it from the copies already written and withholds it in the decision-audit notes and their copies: a dry run by default, `--apply` to rewrite, idempotent. The version history stays the lineage. + +**What else changes.** The data door and the MCP stdio reader serve the two hash columns of the stored-metadata tables in keyed form, or omit them with no provider. A `409` conflict refusal carries keyed values or none. The ObjectQL engine gains a read accessor for the registered provider's keyed digest; it is additive. A member's read of these tables is refused as before. diff --git a/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts b/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts index 92213f79819..760ac65ae93 100644 --- a/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts +++ b/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts @@ -461,7 +461,7 @@ describe('[#7748] the audit trail records the whole lifecycle, not only `save`', // [#21207] The note names which side was present, never a version // token or a stored hash: a copy carries no hash (fork three, ruling A). expect(String(denial.note)).not.toContain('sha256:stale'); - expect(String(denial.note)).toContain('[withheld]'); + expect(String(denial.note)).toContain('(withheld)'); }); // ── the read door ─────────────────────────────────────────────────────── diff --git a/packages/metadata-protocol/src/protocol.package-publish-audit-rows.test.ts b/packages/metadata-protocol/src/protocol.package-publish-audit-rows.test.ts index e4db9ffeff4..29ccb75e6ac 100644 --- a/packages/metadata-protocol/src/protocol.package-publish-audit-rows.test.ts +++ b/packages/metadata-protocol/src/protocol.package-publish-audit-rows.test.ts @@ -980,7 +980,10 @@ describe('[#8594] a refused publish leaves the INNER verdict, in its own vocabul source: 'protocol.publishPackageDrafts', }); // The note names the losing race, which is the fact an author needs. - expect(String(conflict[0].note)).toContain('sha256:advanced_by_a_rival'); + // [#21207] It names it WITHOUT the stored hashes — a copy never + // carries one (fork three, ruling A): each present side is withheld. + expect(String(conflict[0].note)).toBe('expected parent (withheld) but current is (withheld)'); + expect(String(conflict[0].note)).not.toContain('sha256:advanced_by_a_rival'); expect(publishRows(h, 'denied').map((a) => a.code).sort()) .toEqual(['batch_aborted', 'metadata_conflict']); // The batch reports the refusal in its own envelope too. diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 8b3bc649201..2251d604cc7 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -15767,7 +15767,7 @@ export class ObjectStackProtocolImplementation implements * - a caller's token naming no current head: the keyed current head; * - no crypto provider: no value, and the remedy. * - * A side with no served form is `[withheld]`; an absent side is `null`. + * A side with no served form is `(withheld)`; an absent side is `null`. */ private async metadataConflictRefusal(err: ConflictError, subject: string, prefix: string): Promise { const conflict: any = new Error(subject); @@ -15781,7 +15781,7 @@ export class ObjectStackProtocolImplementation implements return conflict; } const digest = this.storedHashDigest(); - const show = (served: string | null | undefined) => (served === undefined ? '[withheld]' : served ?? 'null'); + const show = (served: string | null | undefined) => (served === undefined ? '(withheld)' : served ?? 'null'); const current = await servedContentHash(err.actualHead, digest); if (current !== undefined) conflict.actualHead = current; if (err instanceof InboundVersionConflictError) { @@ -15828,10 +15828,10 @@ export class ObjectStackProtocolImplementation implements // every audit reader), and not keyed either (a copy has no use for a // version token, and a keyed value would die with the key). The // note keeps its sentence and says which side was absent; a value - // is `[withheld]`. `os migrate audit-metadata-bodies` rewrites the + // is `(withheld)`. `os migrate audit-metadata-bodies` rewrites the // notes written before this to exactly this text. - note: `expected parent ${args.expectedParent == null ? 'null' : '[withheld]'} ` - + `but current is ${args.actualHead == null ? 'null' : '[withheld]'}`, + note: `expected parent ${args.expectedParent == null ? 'null' : '(withheld)'} ` + + `but current is ${args.actualHead == null ? 'null' : '(withheld)'}`, }; } diff --git a/packages/objectql/src/publish-package-drafts-response-conformance.test.ts b/packages/objectql/src/publish-package-drafts-response-conformance.test.ts index 63f687725a1..6a753b8549c 100644 --- a/packages/objectql/src/publish-package-drafts-response-conformance.test.ts +++ b/packages/objectql/src/publish-package-drafts-response-conformance.test.ts @@ -28,6 +28,7 @@ * upgrade only when a consumer needs a field) — so the cases here assert it is * carried through unstripped, never its inner shape. */ +import { createHash } from 'node:crypto'; import { describe, it, expect } from 'vitest'; import type { ServiceObject } from '@objectstack/spec/data'; import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; @@ -138,6 +139,12 @@ async function makeProtocol() { // only into a package the registry holds (#20863). Its manifest declares // no namespace, so the ADR-0028 prefix pre-flight is unchanged. engine.registry.installPackage({ id: PKG, name: 'Edu', version: '1.0.0' } as never); + // [#21207] The host's composition-root step (`os serve`, the verify harness): + // a crypto provider. A version token is the provider's keyed digest of the + // stored content hash, so without one the receipts issue none. + engine.setCryptoProvider({ + keyedDigest: async (plain: string) => `hmac-sha256:${createHash('sha256').update(`conformance|${plain}`).digest('hex')}`, + } as never); return new ObjectStackProtocolImplementation(engine); } @@ -216,10 +223,11 @@ describe('publishPackageDrafts response conforms to PublishPackageDraftsResponse expect(parsed.publishedCount).toBe(2); expect(parsed.failedCount).toBe(0); expect(parsed.failed).toEqual([]); - // Every element carries the ADR-0008 OCC token, unstripped. + // Every element carries the ADR-0008 OCC token, unstripped — [#21207] + // in keyed form, never the stored content hash. for (const el of parsed.published) { expect(typeof el.version).toBe('string'); - expect(el.version.length).toBeGreaterThan(0); + expect(el.version).toMatch(/^hmac-sha256:[0-9a-f]{64}$/); } expect(parsed.published.map((e) => e.name).sort()).toEqual(['cases', 'leads']); }); diff --git a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts index e6dfda2ac22..9eca89a0554 100644 --- a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts +++ b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts @@ -107,14 +107,14 @@ export function withoutStoredHashColumns(snapshot: unknown): { changed: boolean; /** * The conflict note with each side's value withheld — `null` kept, anything - * else `[withheld]` — exactly the sentence the protocol writes now; `undefined` + * else `(withheld)` — exactly the sentence the protocol writes now; `undefined` * for a note that is not that sentence or already withholds both sides. */ function withheldConflictNote(note: unknown): string | undefined { if (typeof note !== 'string') return undefined; const match = CONFLICT_NOTE.exec(note); if (!match) return undefined; - const side = (value: string) => (value === 'null' ? 'null' : '[withheld]'); + const side = (value: string) => (value === 'null' ? 'null' : '(withheld)'); const rewritten = `expected parent ${side(match[1] as string)} but current is ${side(match[2] as string)}`; return rewritten === note ? undefined : rewritten; } @@ -158,7 +158,7 @@ function rewriteSerialized( * - `sys_audit_log` / `sys_activity` — the ledger snapshot / activity pair the * writer copied a `sys_metadata_audit` row into. * - * Each value becomes `[withheld]` and a `null` side stays `null` — the sentence + * Each value becomes `(withheld)` and a `null` side stays `null` — the sentence * the protocol writes since this card, so a rewritten note and a new one read * the same. Idempotent. */ diff --git a/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts index afc85fc5106..d242aa7ea50 100644 --- a/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts +++ b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts @@ -34,7 +34,7 @@ const view = (label: string) => JSON.stringify({ name: 'v', type: 'grid', label const metaSnapshot = () => ({ id: 'm1', name: 'v', type: 'view', scope: 'platform', metadata: view('one'), checksum: HASH }); const historySnapshot = () => ({ id: 'h1', name: 'v', type: 'view', metadata: view('one'), checksum: HASH, previous_checksum: PARENT }); const conflictNote = `expected parent ${PARENT} but current is ${HASH}`; -const WITHHELD_NOTE = 'expected parent [withheld] but current is [withheld]'; +const WITHHELD_NOTE = 'expected parent (withheld) but current is (withheld)'; describe('planAuditRowPatch / planActivityRowPatch — the hash columns leave the copy', () => { it('a sys_metadata create snapshot loses its checksum, and nothing else', () => { @@ -89,7 +89,7 @@ describe('planDecisionNotePatch — the decision-audit note and its copies', () it('keeps a null side as null', () => { const patch = planDecisionNotePatch('sys_metadata_audit', { id: 'd2', code: 'metadata_conflict', note: `expected parent null but current is ${HASH}` }); - expect(patch).toEqual({ note: 'expected parent null but current is [withheld]' }); + expect(patch).toEqual({ note: 'expected parent null but current is (withheld)' }); }); it('leaves every other note alone, and is idempotent', () => { diff --git a/packages/rest/src/rest-server.ts b/packages/rest/src/rest-server.ts index 765ce8c9990..53ed10a9399 100644 --- a/packages/rest/src/rest-server.ts +++ b/packages/rest/src/rest-server.ts @@ -6981,10 +6981,13 @@ export class RestServer { : body; // Opt-in OCC under ADR-0008 PR-10d.3: callers (Studio, - // CLI) may set `If-Match: ` to enforce that - // the overlay row has not advanced since they last read - // it. A `null`/empty body or no header preserves the - // legacy last-write-wins behaviour. + // CLI) may set `If-Match` to the version token a receipt + // served, to enforce that the overlay row has not advanced + // since they last read it. A `null`/empty body or no header + // preserves the legacy last-write-wins behaviour. [#21207] + // The token is the crypto provider's keyed digest of the + // stored content hash, never the hash itself; the protocol + // compares it in that form, so it passes through here as sent. const ifMatchHeader = req.headers?.['if-match'] ?? req.headers?.['If-Match']; const parentVersion = typeof ifMatchHeader === 'string' ? ifMatchHeader.replace(/^"|"$/g, '') // strip ETag-style quotes From 25c922482e8032dd4f39b77d9249c33bb394ec53 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 14:11:57 +0000 Subject: [PATCH 05/11] fix(metadata-protocol,mcp,plugin-audit,service-analytics)!: the history change note that quotes a stored hash joins the family (#21207) A draft promotion with no message of its own recorded the draft's stored content hash in the history row's change note, served by the history read, the data door and the MCP stdio reader and copied by the audit writer. The publish door now states a hash-free message; a stored note is served with each quoted hash keyed (withheld with no provider), is never evaluated, and its copies withhold the quote at write time and at rest. Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .changeset/21207-keyed-served-content-hash.md | 2 +- ...io-data-bridge.stored-content-hash.test.ts | 28 ++++++++++ packages/mcp/src/stdio-data-bridge.ts | 33 ++++++++++-- .../src/metadata-redaction.ts | 54 ++++++++++++++++--- ...ocol.data-door-stored-content-hash.test.ts | 45 ++++++++++++++++ .../src/protocol.served-content-hash.test.ts | 30 +++++++++++ packages/metadata-protocol/src/protocol.ts | 12 ++++- .../stored-metadata-body-family.pin.test.ts | 17 +++++- .../plugin-audit/src/audit-writers.test.ts | 14 +++++ .../plugins/plugin-audit/src/audit-writers.ts | 9 +++- .../src/stored-metadata-body-migration.ts | 24 ++++++++- .../stored-metadata-hash-migration.test.ts | 12 +++++ .../src/stored-metadata-body-refusal.test.ts | 11 ++++ .../src/stored-metadata-body-refusal.ts | 12 ++++- 14 files changed, 283 insertions(+), 20 deletions(-) diff --git a/.changeset/21207-keyed-served-content-hash.md b/.changeset/21207-keyed-served-content-hash.md index 1407cdc61cb..a8734ff4b58 100644 --- a/.changeset/21207-keyed-served-content-hash.md +++ b/.changeset/21207-keyed-served-content-hash.md @@ -18,7 +18,7 @@ Clause-②: yes (narrowing) **Three things change for callers and operators.** 1. **A held version token gets one `409 METADATA_CONFLICT`.** Every door that hands out a metadata version token — the save, publish, package-publish and rollback receipts and the history read — now hands out the crypto provider's keyed digest of the stored hash instead of the hash itself, and the save and reset doors compare a token they are sent in that same form. A token a client held from before the upgrade is refused once; take the token from the next read or receipt and retry. A deployment that registers no crypto provider issues no token — the receipts' version is empty and the history read serves none — and refuses a sent token with the same `409`; send the write without one, or register a provider. -2. **Filter, sort and group on the two stored content-hash columns now answer `400 INVALID_FIELD`** — on the generic data door, the MCP stdio reader and the analytics door, before the engine runs. A data-door search over the two stored-metadata tables no longer scans those columns or the stored body column, and an explicit search-field list naming one answers the same `400`. Every other column of the two tables is served, filtered, sorted and grouped as before, and every other object is unchanged. +2. **Filter, sort and group on the two stored content-hash columns, and on the version history's change note, now answer `400 INVALID_FIELD`** — on the generic data door, the MCP stdio reader and the analytics door, before the engine runs. The change note is included because a draft promotion that stated no message of its own recorded the draft's stored hash in it; the publish door now always states a hash-free message, and a note written before this release is served with the quoted hash in keyed form. A data-door search over the two stored-metadata tables no longer scans those columns or the stored body column, and an explicit search-field list naming one answers the same `400`. Every other column of the two tables is served, filtered, sorted and grouped as before, and every other object is unchanged. 3. **Operators run `os migrate audit-metadata-bodies` once after upgrading, dry run first.** The audit ledger, the activity feed and the metadata decision-audit trail no longer copy the stored hash. The extended command drops it from the copies already written and withholds it in the decision-audit notes and their copies: a dry run by default, `--apply` to rewrite, idempotent. The version history stays the lineage. **What else changes.** The data door and the MCP stdio reader serve the two hash columns of the stored-metadata tables in keyed form, or omit them with no provider. A `409` conflict refusal carries keyed values or none. The ObjectQL engine gains a read accessor for the registered provider's keyed digest; it is additive. A member's read of these tables is refused as before. diff --git a/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts b/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts index 9f865f04664..e956f69faf5 100644 --- a/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts +++ b/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts @@ -299,3 +299,31 @@ describe('[#21207] stdio record resource: the keyed hash, over the transport', ( expect(h.name).toBe('v'); }); }); + +describe('[#21207] stdio reader: the history change note that quotes a stored hash', () => { + const QUOTED = `sha256:${'4'.repeat(64)}`; + const NOTE_ROW = { id: 'h_note', type: 'view', name: 'v', metadata: '{"name":"v"}', checksum: QUOTED, previous_checksum: null, change_note: `publish draft (hash ${QUOTED})` }; + + it('is served with the quote keyed, and withheld with no provider', async () => { + TABLES.sys_metadata_history!.push(NOTE_ROW); + try { + const keyed = (await bridgeAs(ADMIN).bridge.get('sys_metadata_history', 'h_note')) as Record; + expect(keyed.change_note).toBe(`publish draft (hash ${await keyedDigest(QUOTED)})`); + const bare = (await bridgeAs(ADMIN, { provider: false }).bridge.get('sys_metadata_history', 'h_note')) as Record; + expect(bare.change_note).toBe('publish draft (hash (withheld))'); + } finally { + TABLES.sys_metadata_history!.pop(); + } + }); + + it('a filter or sort on the change note is refused (INVALID_FIELD / 400)', async () => { + const { bridge, engine } = bridgeAs(ADMIN); + for (const run of [ + () => bridge.query('sys_metadata_history', { where: { change_note: 'x' } }), + () => bridge.query('sys_metadata_history', { orderBy: [{ field: 'change_note', order: 'asc' }] }), + ]) { + await expect(run()).rejects.toMatchObject({ code: 'INVALID_FIELD', status: 400, field: 'change_note' }); + } + expect(engine.find).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/mcp/src/stdio-data-bridge.ts b/packages/mcp/src/stdio-data-bridge.ts index 5d8c4aa1ea3..db03321184f 100644 --- a/packages/mcp/src/stdio-data-bridge.ts +++ b/packages/mcp/src/stdio-data-bridge.ts @@ -451,11 +451,33 @@ export function storedMetadataBodyRefusal( */ export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); -/** The content-hash column a field reference reaches — the column, or a dotted path headed by it. */ +/** + * [#21207] The history table's change note, which can QUOTE a stored content + * hash (`publish draft (hash …)` on rows written before the publish door stated + * its own message): served with each quote in its served form, never evaluated. + */ +export const STORED_METADATA_HASH_NOTE_COLUMN = 'change_note'; + +/** Every column that holds or can quote a stored content hash. */ +const HASH_BEARING_COLUMNS: readonly string[] = [...STORED_METADATA_HASH_COLUMNS, STORED_METADATA_HASH_NOTE_COLUMN]; + +/** An unkeyed content hash quoted in free text (the keyed form's `hmac-sha256:` prefix is not one). */ +const QUOTED_STORED_HASH = /(? { + const quoted = text.match(QUOTED_STORED_HASH); + if (!quoted) return text; + const served = new Map(); + for (const stored of new Set(quoted)) served.set(stored, digest ? await digest(stored) : '(withheld)'); + return text.replace(QUOTED_STORED_HASH, (stored) => served.get(stored) as string); +} + +/** The hash-bearing column a field reference reaches — the column, or a dotted path headed by it. */ function hashColumnOf(field: unknown): string | undefined { if (typeof field !== 'string') return undefined; const head = field.split('.')[0] as string; - return STORED_METADATA_HASH_COLUMNS.includes(head) ? head : undefined; + return HASH_BEARING_COLUMNS.includes(head) ? head : undefined; } /** @@ -480,7 +502,8 @@ export function storedMetadataHashRefusal( const make = (param: 'groupBy' | 'filter' | 'sort', column: string): McpStoredMetadataBodyRefusal => { const doing = param === 'groupBy' ? 'group' : param; const err = new Error( - `Cannot ${doing} '${object}' by '${column}' (${param}): the query was not run. The '${column}' column holds ` + `Cannot ${doing} '${object}' by '${column}' (${param}): the query was not run. The '${column}' column ` + + `${column === STORED_METADATA_HASH_NOTE_COLUMN ? 'can quote' : 'holds'} ` + `the stored content hash of a metadata body, computed over withheld credential material too, so this door ` + `serves it only in keyed form; ${param === 'filter' ? 'a filter on it compares a guess against the stored hash row by row, which confirms the guess' @@ -531,7 +554,7 @@ export async function serveStoredMetadataHashes( ): Promise { if (!isStoredMetadataBodyObject(object) || !row || typeof row !== 'object' || Array.isArray(row)) return row; const record = row as Record; - if (!STORED_METADATA_HASH_COLUMNS.some((column) => column in record)) return row; + if (!HASH_BEARING_COLUMNS.some((column) => column in record)) return row; const out: Record = { ...record }; for (const column of STORED_METADATA_HASH_COLUMNS) { if (!(column in out)) continue; @@ -539,6 +562,8 @@ export async function serveStoredMetadataHashes( if (!digest || (stored !== null && typeof stored !== 'string')) delete out[column]; else out[column] = stored === null ? null : await digest(stored); } + const note = out[STORED_METADATA_HASH_NOTE_COLUMN]; + if (typeof note === 'string') out[STORED_METADATA_HASH_NOTE_COLUMN] = await serveStoredHashTokens(note, digest); return out as T; } diff --git a/packages/metadata-protocol/src/metadata-redaction.ts b/packages/metadata-protocol/src/metadata-redaction.ts index f26f6ac5e45..1c62629fcf2 100644 --- a/packages/metadata-protocol/src/metadata-redaction.ts +++ b/packages/metadata-protocol/src/metadata-redaction.ts @@ -807,6 +807,39 @@ export function storedMetadataBodyPredicateRefusal( */ export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); +/** + * [#21207] The history table's free-text change note — which can QUOTE a stored + * content hash: with no message of its own, a draft's promotion recorded + * `publish draft (hash )`. The protocol now always + * states a hash-free message, so no new row carries one; rows already written + * do, so the column is served with each quoted hash in its served form + * ({@link serveStoredHashTokens}) and is never evaluated, exactly as the hash + * columns are. + */ +export const STORED_METADATA_HASH_NOTE_COLUMN = 'change_note'; + +/** Every column of a stored-metadata table that holds or can quote a stored content hash. */ +export const STORED_METADATA_HASH_BEARING_COLUMNS: readonly string[] = Object.freeze([ + ...STORED_METADATA_HASH_COLUMNS, + STORED_METADATA_HASH_NOTE_COLUMN, +]); + +/** An unkeyed content hash quoted in free text (the keyed form's `hmac-sha256:` prefix is not one). */ +const QUOTED_STORED_HASH = /(? { + const quoted = text.match(QUOTED_STORED_HASH); + if (!quoted) return text; + const served = new Map(); + for (const stored of new Set(quoted)) served.set(stored, digest ? await digest(stored) : '(withheld)'); + return text.replace(QUOTED_STORED_HASH, (stored) => served.get(stored) as string); +} + /** The keyed-digest primitive of the registered crypto provider (`ICryptoProvider.keyedDigest`). */ export type StoredHashDigest = (plain: string) => Promise; @@ -842,7 +875,7 @@ export async function serveStoredMetadataHashColumns( digest: StoredHashDigest | undefined, ): Promise { if (!isStoredMetadataBodyObject(object) || !isPlainRecord(row)) return row; - if (!STORED_METADATA_HASH_COLUMNS.some((column) => column in row)) return row; + if (!STORED_METADATA_HASH_BEARING_COLUMNS.some((column) => column in row)) return row; const out: Record = { ...row }; for (const column of STORED_METADATA_HASH_COLUMNS) { if (!(column in out)) continue; @@ -852,6 +885,8 @@ export async function serveStoredMetadataHashColumns( if (served === undefined) delete out[column]; else out[column] = served; } + const note = out[STORED_METADATA_HASH_NOTE_COLUMN]; + if (typeof note === 'string') out[STORED_METADATA_HASH_NOTE_COLUMN] = await serveStoredHashTokens(note, digest); return out as T; } @@ -869,7 +904,7 @@ export async function serveStoredMetadataHashColumnRows( function hashColumnOf(field: unknown): string | undefined { if (typeof field !== 'string') return undefined; const head = field.split('.')[0] as string; - return STORED_METADATA_HASH_COLUMNS.includes(head) ? head : undefined; + return STORED_METADATA_HASH_BEARING_COLUMNS.includes(head) ? head : undefined; } /** The columns a refusal on these tables points the caller at instead. */ @@ -877,7 +912,8 @@ const USABLE_COLUMNS = `'${STORED_TYPE_COLUMN}', 'name', 'state' or another scal /** * [#21207] The data door's refusal to EVALUATE a content-hash column of a - * stored-metadata table — a grouping (whose keys would serve the stored + * stored-metadata table — or the history table's change note, which can quote + * one ({@link STORED_METADATA_HASH_BEARING_COLUMNS}) — a grouping (whose keys would serve the stored * values), a filter (an online verifier: a guessed hash matches exactly one * row) or a sort (an order over the same values) — or `undefined` when none is * named. Maintainer ruling A on #21207's second execution fork. @@ -902,7 +938,9 @@ export function storedMetadataHashEvaluateRefusal( : 'a sort on it orders by the stored values'; const err: any = new Error( `Cannot ${doing} '${object}' by '${column}' (${position ?? param}): the query was not run. The ` - + `'${column}' column holds the stored content hash of a metadata body, computed over withheld ` + + `'${column}' column ${column === STORED_METADATA_HASH_NOTE_COLUMN + ? 'can quote the stored content hash of a metadata body' + : 'holds the stored content hash of a metadata body'}, computed over withheld ` + `credential material too, so this door serves it only in keyed form; ${why}. ` + `${doing === 'group' ? 'Group' : doing === 'filter' ? 'Filter' : 'Sort'} by ${USABLE_COLUMNS} instead.`, ); @@ -935,7 +973,7 @@ export function storedMetadataHashEvaluateRefusal( /** * [#21207] The columns of a stored-metadata table a `search` never scans: the - * body column and the content-hash columns. A search is a substring filter + * body column, the content-hash columns and the change note that can quote one. A search is a substring filter * evaluated server-side over every scanned column, so over these columns it is * the same verifier a filter is — over the stored hash, and over the stored * body (a withheld credential rebuilt by prefix probing) — and the engine's @@ -944,7 +982,7 @@ export function storedMetadataHashEvaluateRefusal( */ export const STORED_METADATA_UNSEARCHABLE_COLUMNS: readonly string[] = Object.freeze([ STORED_BODY_COLUMN, - ...STORED_METADATA_HASH_COLUMNS, + ...STORED_METADATA_HASH_BEARING_COLUMNS, ]); /** @@ -965,7 +1003,9 @@ export function storedMetadataSearchRefusal( `Cannot search '${object}' in '${column}' (${param}): the query was not run. A search evaluates ` + `every column it scans row by row, and the '${column}' column holds ${column === STORED_BODY_COLUMN ? 'a stored metadata body with credential material this door withholds' - : 'the stored content hash of a metadata body, which this door serves only in keyed form'}, so a ` + : column === STORED_METADATA_HASH_NOTE_COLUMN + ? 'a change note that can quote a stored content hash, which this door serves only in keyed form' + : 'the stored content hash of a metadata body, which this door serves only in keyed form'}, so a ` + `search over it would answer guesses about withheld values. Search ${USABLE_COLUMNS} instead.`, ); err.code = 'INVALID_FIELD'; diff --git a/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts b/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts index e5e640fe3fa..a30205f478c 100644 --- a/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts +++ b/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts @@ -256,3 +256,48 @@ describe('[#21207] data door — a search never scans the hash or body columns o expect(find.mock.calls[0]![1].searchFields).toBeUndefined(); }); }); + +describe('[#21207] data door — the history change note that quotes a stored hash', () => { + const QUOTED = hashSpec({ quoted: true }); + const NOTE_ROWS = [{ id: 'h_note', type: 'view', name: 'all_tasks', version: 3, operation_type: 'publish', metadata: '{}', checksum: QUOTED, previous_checksum: null, change_note: `publish draft (hash ${QUOTED})` }]; + + it('is served with the quote keyed, and withheld with no provider', async () => { + const saved = [...HISTORY_ROWS]; + HISTORY_ROWS.push(...(NOTE_ROWS as any)); + try { + const { p } = makeProtocol(); + const got: any = await p.getData({ object: 'sys_metadata_history', id: 'h_note' }); + expect(got.record.change_note).toBe(`publish draft (hash ${await keyedDigest(QUOTED)})`); + const bare = makeProtocol({ provider: false }); + const withheld: any = await bare.p.findData({ object: 'sys_metadata_history', query: {} }); + expect(byId(withheld.records, 'h_note').change_note).toBe('publish draft (hash (withheld))'); + expect(JSON.stringify(withheld.records)).not.toContain(QUOTED); + } finally { + HISTORY_ROWS.length = 0; + HISTORY_ROWS.push(...saved); + } + }); + + for (const [label, query] of [ + ['filter', { filter: JSON.stringify({ change_note: 'x' }) }], + ['sort', { sort: 'change_note' }], + ['group', { groupBy: ['change_note'], aggregations: [{ function: 'count', alias: 'n' }] }], + ['explicit search fields', { search: 'abc', searchFields: 'change_note' }], + ] as const) { + it(`${label} on the change note is refused (INVALID_FIELD / 400)`, async () => { + const { p, find, aggregate } = makeProtocol(); + const err = await refusal(() => p.findData({ object: 'sys_metadata_history', query: query as any })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe('change_note'); + expect(find).not.toHaveBeenCalled(); + expect(aggregate).not.toHaveBeenCalled(); + }); + } + + it('an implicit search does not scan the change note', async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'sys_metadata_history', query: { search: 'abc' } }); + expect(find.mock.calls[0]![1].searchFields).not.toContain('change_note'); + }); +}); diff --git a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts index 6a6e155a971..228710a20b1 100644 --- a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts +++ b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts @@ -359,3 +359,33 @@ describe('[#21207] no crypto provider: nothing served, every inbound token refus expect(activeHash(h)).toBe(raw); }); }); + +describe('[#21207] a change note that quotes a stored hash', () => { + it('the publish door writes a note that quotes no hash', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + await p.saveMetaItem({ ...ref, item: viewBody('staged'), mode: 'draft' } as any); + await p.publishMetaItem({ ...ref } as any); + const notes = h.historyRows.map((r) => r.change_note).filter((n) => typeof n === 'string') as string[]; + expect(notes.length).toBeGreaterThan(0); + for (const note of notes) expect(note).not.toMatch(SHA256); + }); + + it('the history read serves a stored note\'s quoted hash keyed, and withheld with no provider', async () => { + for (const provider of [true, false]) { + const h = makeEngine({ provider }); + const p = new ObjectStackProtocolImplementation(h.engine); + await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const stored = activeHash(h); + // A row written before the publish door stated its own message. + for (const row of h.historyRows) row.change_note = `publish draft (hash ${stored})`; + + const { events } = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + expect(events.length).toBeGreaterThan(0); + for (const ev of events) { + expect(ev.message).toBe(provider ? `publish draft (hash ${await keyedDigest(stored)})` : 'publish draft (hash (withheld))'); + } + expectNoStoredHash(JSON.stringify(events), storedHashes(h)); + } + }); +}); diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index d19051beff7..a3a912264bf 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -196,6 +196,7 @@ import { // evaluated — see `STORED_METADATA_HASH_COLUMNS`. isStoredMetadataBodyObject, servedContentHash, + serveStoredHashTokens, serveStoredMetadataHashColumnRows, serveStoredMetadataHashColumns, STORED_METADATA_UNSEARCHABLE_COLUMNS, @@ -18799,12 +18800,16 @@ export class ObjectStackProtocolImplementation implements // the same value the write receipts hand out, so an event still names // the token a caller holds — and `null` with no crypto provider (a // delete event's own `null` is kept either way). + // The event's message is the row's change note, which can QUOTE a stored + // hash (`publish draft (hash …)` on rows written before the publish door + // stated its own message) — each quote is served the same way. const digest = this.storedHashDigest(); for await (const ev of repo.history(ref, opts)) { events.push({ ...ev, hash: (await servedContentHash(ev.hash, digest)) ?? null, parentHash: (await servedContentHash(ev.parentHash, digest)) ?? null, + ...(typeof ev.message === 'string' ? { message: await serveStoredHashTokens(ev.message, digest) } : {}), }); } return { events }; @@ -19340,7 +19345,12 @@ export class ObjectStackProtocolImplementation implements // #4556 — NULL, not 'system', for an actor-less publish. actor: request.actor ?? null, source: 'protocol.publishMetaItem', - ...(request.message ? { message: request.message } : {}), + // [#21207] Always a message of the caller's or this door's own: + // left unstated, the repository records `publish draft (hash …)`, + // quoting the draft's stored content hash into the history row's + // change note — served to every history reader and copied by the + // audit writer. This door's default says what happened without it. + message: request.message || 'publish draft', intent, // [#8907] Spread, not `packageId: request.packageId`: `null` is // a meaningful scope (the unbound row) and `undefined` means diff --git a/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts b/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts index 7d7ee7644b7..7647c50e02e 100644 --- a/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts +++ b/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts @@ -111,6 +111,18 @@ const FAMILY_SURFACES = [ disposition: 'withheld', pin: '@objectstack/plugin-audit', }, + // A stored row's change note can QUOTE a stored hash (a draft promotion with + // no message of its own recorded one); found by this card's measurement. + { + surface: 'history change note quoting a stored hash: /meta history message, data door, MCP stdio reader', + disposition: 'keyed', + pin: 'this package (protocol.served-content-hash.test.ts, protocol.data-door-stored-content-hash.test.ts) + @objectstack/mcp', + }, + { + surface: 'history change note: group / filter / sort / search, and as an analytics member', + disposition: 'refuses', + pin: 'this package (protocol.data-door-stored-content-hash.test.ts) + @objectstack/mcp + @objectstack/service-analytics', + }, ] as const; /** A stored datasource body as it sits in the `metadata` column: serialized JSON with credential material. */ @@ -180,8 +192,9 @@ describe('[#21120] stored-metadata-body family — the data door exposes only co } // One row per the three local data-door surfaces + five cross-package ones // (audit, analytics, realtime, and the MCP stdio reader's seam and refusal), - // + [#21207] the thirteen surfaces of the stored content hash. - expect(FAMILY_SURFACES).toHaveLength(21); + // + [#21207] the thirteen surfaces of the stored content hash, and the two + // of the history change note that can quote one. + expect(FAMILY_SURFACES).toHaveLength(23); }); }); diff --git a/packages/plugins/plugin-audit/src/audit-writers.test.ts b/packages/plugins/plugin-audit/src/audit-writers.test.ts index 3c1d797fdc3..625dbf67dc9 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.test.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.test.ts @@ -2351,6 +2351,20 @@ describe('[#21207] stored metadata copies carry no content hash', () => { for (const c of created) hashFree(JSON.stringify(c.row)); }); + it('a history append whose change note quotes a hash: the copy keeps the note and withholds the quote', async () => { + const { engine, fire, created } = makeEngine({ ...SCHEMAS, sys_metadata_history: [...SCHEMAS.sys_metadata_history, 'change_note'] }); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterInsert', { + object: 'sys_metadata_history', + input: { id: 'h-2' }, + result: { id: 'h-2', name: 'v', type: 'view', metadata: view('one'), checksum: NEXT, change_note: `publish draft (hash ${NEXT})` }, + session: { userId: 'admin-1' }, + }); + for (const c of created) hashFree(JSON.stringify(c.row)); + const audit = created.find((c) => c.object === 'sys_audit_log'); + expect(JSON.parse(audit!.row.new_value).change_note).toBe('publish draft (hash (withheld))'); + }); + it('control: another object keeps its own checksum column in the copy', async () => { const { engine, fire, created } = makeEngine(SCHEMAS); installAuditWriters(engine as any, 'test.audit'); diff --git a/packages/plugins/plugin-audit/src/audit-writers.ts b/packages/plugins/plugin-audit/src/audit-writers.ts index 82642c4db9d..7a907addcda 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.ts @@ -45,7 +45,11 @@ import { } from '@objectstack/spec/kernel'; // [#21207] The same rows' stored content-hash columns — defined beside the // at-rest rewrite that withholds the copies already written. -import { STORED_METADATA_HASH_COLUMNS } from './stored-metadata-body-migration.js'; +import { + STORED_METADATA_HASH_COLUMNS, + STORED_METADATA_HASH_NOTE_COLUMN, + withheldStoredHashTokens, +} from './stored-metadata-body-migration.js'; // [commit 1408fe385 / #10101] The platform-row organization resolver, imported rather // than owned. It started life in THIS file (commit 1408fe385, honouring #8287's ruling) // and was promoted to `@objectstack/metadata-core` by the maintainer ruling @@ -1432,6 +1436,9 @@ export function installAuditWriters( // has no use for a version token, and the history table stays the lineage. if (isStoredMetadataBodyObject(objectName)) { for (const column of STORED_METADATA_HASH_COLUMNS) delete out[column]; + // …and a change note that quotes one keeps its words, not the hash. + const note = out[STORED_METADATA_HASH_NOTE_COLUMN]; + if (typeof note === 'string') out[STORED_METADATA_HASH_NOTE_COLUMN] = withheldStoredHashTokens(note); } if (dropComputed) { const defs = getFieldDefs(objectName); diff --git a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts index 9eca89a0554..43d2b5aaa87 100644 --- a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts +++ b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts @@ -81,6 +81,21 @@ export const STORED_METADATA_BODY_AUDIT_OBJECTS = ['sys_audit_log', 'sys_activit */ export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); +/** + * [#21207] The history table's change note, which can QUOTE a stored content + * hash (`publish draft (hash …)` on rows written before the publish door stated + * its own message). A copy keeps the note and withholds each quote. + */ +export const STORED_METADATA_HASH_NOTE_COLUMN = 'change_note'; + +/** An unkeyed content hash quoted in free text (the keyed form's `hmac-sha256:` prefix is not one). */ +const QUOTED_STORED_HASH = /(?; - if (!STORED_METADATA_HASH_COLUMNS.some((column) => column in record)) return { changed: false, value: snapshot }; + const note = record[STORED_METADATA_HASH_NOTE_COLUMN]; + const withheldNote = typeof note === 'string' ? withheldStoredHashTokens(note) : note; + const hasHash = STORED_METADATA_HASH_COLUMNS.some((column) => column in record); + if (!hasHash && withheldNote === note) return { changed: false, value: snapshot }; const out: Record = { ...record }; for (const column of STORED_METADATA_HASH_COLUMNS) delete out[column]; + if (withheldNote !== note) out[STORED_METADATA_HASH_NOTE_COLUMN] = withheldNote; return { changed: true, value: out }; } diff --git a/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts index d242aa7ea50..abc63cbbb61 100644 --- a/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts +++ b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts @@ -175,3 +175,15 @@ describe('migrateStoredMetadataBodyCopies — the content-hash copies (driven)', expect(second.rewritten).toBe(0); }); }); + +describe('the history change note that quotes a stored hash (#21207)', () => { + it('a copied history snapshot keeps its note and withholds the quote, idempotently', () => { + const snapshot = { ...historySnapshot(), change_note: `publish draft (hash ${HASH})` }; + const row = { id: 'a7', object_name: 'sys_metadata_history', record_id: 'h1', new_value: JSON.stringify(snapshot), old_value: null }; + const patch = planAuditRowPatch(row)!; + const rewritten = JSON.parse(String(patch.new_value)); + expect(rewritten.change_note).toBe('publish draft (hash (withheld))'); + expect(String(patch.new_value)).not.toMatch(SHA256); + expect(planAuditRowPatch({ ...row, new_value: patch.new_value })).toBeNull(); + }); +}); diff --git a/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts b/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts index 99e88113d76..033306806ef 100644 --- a/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts +++ b/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts @@ -111,3 +111,14 @@ describe('storedMetadataBodyAnalyticsRefusal — the content-hash columns (#2120 expect(storedMetadataBodyAnalyticsRefusal([field('file_blob', 'previous_checksum', 'predicate')])).toBeUndefined(); }); }); + +describe('storedMetadataBodyAnalyticsRefusal — the history change note (#21207)', () => { + it('refuses the change note, which can quote a stored hash, in either role', () => { + for (const role of ['aggregate', 'predicate'] as const) { + const err = storedMetadataBodyAnalyticsRefusal([field('sys_metadata_history', 'change_note', role)]) as any; + expect(err?.code).toBe('INVALID_FIELD'); + expect(err?.status).toBe(400); + expect(err?.field).toBe('change_note'); + } + }); +}); diff --git a/packages/services/service-analytics/src/stored-metadata-body-refusal.ts b/packages/services/service-analytics/src/stored-metadata-body-refusal.ts index 6b4149c4674..9be677a07df 100644 --- a/packages/services/service-analytics/src/stored-metadata-body-refusal.ts +++ b/packages/services/service-analytics/src/stored-metadata-body-refusal.ts @@ -55,6 +55,13 @@ import { isStoredMetadataBodyObject, STORED_METADATA_BODY_COLUMN } from '@object * the object definitions. */ export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); + +/** + * [#21207] The history table's change note, which can QUOTE a stored content + * hash (`publish draft (hash …)` on rows written before the publish door stated + * its own message) — refused as a member for the same reason. + */ +export const STORED_METADATA_HASH_NOTE_COLUMN = 'change_note'; import type { StandardErrorCode } from '@objectstack/spec/api'; import type { NamedField, NamedRead } from './field-read-admission.js'; @@ -76,7 +83,7 @@ export function storedMetadataBodyAnalyticsRefusal( if ('expression' in f) continue; if (!isStoredMetadataBodyObject(f.object)) continue; const isBody = f.field === STORED_METADATA_BODY_COLUMN; - if (!isBody && !STORED_METADATA_HASH_COLUMNS.includes(f.field)) continue; + if (!isBody && !STORED_METADATA_HASH_COLUMNS.includes(f.field) && f.field !== STORED_METADATA_HASH_NOTE_COLUMN) continue; const param = f.role === 'aggregate' ? 'dimensions' : 'where'; const err = new Error( isBody @@ -86,7 +93,8 @@ export function storedMetadataBodyAnalyticsRefusal( + `stored body (a group key that cannot be projected, or a filter oracle that rebuilds a withheld value ` + `by probing). Group, filter or sort by 'type', 'name' or another scalar column instead.` // [#21207] A stored content-hash column, in the same envelope. - : `Cannot query '${f.object}' by '${f.field}': the query was not run. The ${f.field} column holds the ` + : `Cannot query '${f.object}' by '${f.field}': the query was not run. The ${f.field} column ` + + `${f.field === STORED_METADATA_HASH_NOTE_COLUMN ? 'can quote' : 'holds'} the ` + `stored content hash of a metadata body, computed over withheld credential material too, so every read ` + `exit serves it only in keyed form; grouping, aggregating, filtering or sorting by it would evaluate the ` + `stored hash (a group key that serves it, or a filter that confirms a guessed hash). Group, filter or ` From 27b53321f1b35f72d716beebcf0f9ccd6b22e20a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 14:34:31 +0000 Subject: [PATCH 06/11] fix(plugin-audit): a copied decision-audit note withholds the hashes it quotes (#21207) The extended migration's rewrite of a conflict note is itself an audited update, and its ledger copy carried the old note's hashes back into the ledger: one apply left one copy to rewrite. The writer now withholds a quoted stored hash in any copied decision-audit note, so one apply converges. Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .../plugins/plugin-audit/src/audit-writers.test.ts | 14 ++++++++++++++ packages/plugins/plugin-audit/src/audit-writers.ts | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/packages/plugins/plugin-audit/src/audit-writers.test.ts b/packages/plugins/plugin-audit/src/audit-writers.test.ts index 625dbf67dc9..5048dc9af07 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.test.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.test.ts @@ -2365,6 +2365,20 @@ describe('[#21207] stored metadata copies carry no content hash', () => { expect(JSON.parse(audit!.row.new_value).change_note).toBe('publish draft (hash (withheld))'); }); + it('a decision-audit note that quotes a hash (its rewrite, or a row written before) is copied withheld', async () => { + const { engine, fire, created } = makeEngine({ ...SCHEMAS, sys_metadata_audit: ['id', 'code', 'note'] }); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterUpdate', { + object: 'sys_metadata_audit', + input: { id: 'd-1', data: { note: 'expected parent (withheld) but current is (withheld)' } }, + previous: { id: 'd-1', code: 'metadata_conflict', note: `expected parent ${PARENT} but current is ${HASH}` }, + result: { id: 'd-1', code: 'metadata_conflict', note: 'expected parent (withheld) but current is (withheld)' }, + session: {}, + }); + expect(created.length).toBeGreaterThan(0); + for (const c of created) hashFree(JSON.stringify(c.row)); + }); + it('control: another object keeps its own checksum column in the copy', async () => { const { engine, fire, created } = makeEngine(SCHEMAS); installAuditWriters(engine as any, 'test.audit'); diff --git a/packages/plugins/plugin-audit/src/audit-writers.ts b/packages/plugins/plugin-audit/src/audit-writers.ts index 7a907addcda..dbb8419561e 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.ts @@ -46,6 +46,7 @@ import { // [#21207] The same rows' stored content-hash columns — defined beside the // at-rest rewrite that withholds the copies already written. import { + METADATA_DECISION_AUDIT_OBJECT, STORED_METADATA_HASH_COLUMNS, STORED_METADATA_HASH_NOTE_COLUMN, withheldStoredHashTokens, @@ -1440,6 +1441,13 @@ export function installAuditWriters( const note = out[STORED_METADATA_HASH_NOTE_COLUMN]; if (typeof note === 'string') out[STORED_METADATA_HASH_NOTE_COLUMN] = withheldStoredHashTokens(note); } + // The same for a decision-audit row's note: a conflict note written before + // the protocol withheld its hashes — and the rewrite of one by + // `os migrate audit-metadata-bodies`, whose own ledger copy would otherwise + // carry the old note's hashes straight back into the ledger. + if (objectName === METADATA_DECISION_AUDIT_OBJECT && typeof out.note === 'string') { + out.note = withheldStoredHashTokens(out.note); + } if (dropComputed) { const defs = getFieldDefs(objectName); for (const key of Object.keys(out)) { From 9d3a09c81a07a8f08be0c3c81f26fac55d9e0e26 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 15:02:32 +0000 Subject: [PATCH 07/11] chore(plugin-audit,metadata-protocol): gate hygiene for the content-hash family (#21207) Record the new pinned engine doubles in the engine-double ledger, read the decision-audit code column through an operator-form predicate (a read, not a stamp), and mark the persisted audit vocabulary in the new pins. Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .../src/protocol.served-content-hash.test.ts | 2 +- .../plugin-audit/src/audit-writers.test.ts | 2 ++ .../src/stored-metadata-body-migration.ts | 6 +++- .../stored-metadata-hash-migration.test.ts | 14 +++++---- scripts/engine-double-contract.pinned.json | 30 +++++++++++++++++++ 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts index 228710a20b1..aa6e578ce1f 100644 --- a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts +++ b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts @@ -299,7 +299,7 @@ describe('[#21207] inbound version tokens are compared in keyed form', () => { await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); await rejection(() => p.saveMetaItem({ ...ref, item: viewBody('lost'), parentVersion: v1.version } as any)); - const denials = h.auditRows.filter((a) => a.code === 'metadata_conflict'); + const denials = h.auditRows.filter((a) => a.code === 'metadata_conflict'); // adr0112-ok: D6b persisted audit column expect(denials).toHaveLength(1); const note = String(denials[0]!.note); expect(note).not.toMatch(SHA256); diff --git a/packages/plugins/plugin-audit/src/audit-writers.test.ts b/packages/plugins/plugin-audit/src/audit-writers.test.ts index 5048dc9af07..868cfcfb1cf 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.test.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.test.ts @@ -2371,7 +2371,9 @@ describe('[#21207] stored metadata copies carry no content hash', () => { await fire('afterUpdate', { object: 'sys_metadata_audit', input: { id: 'd-1', data: { note: 'expected parent (withheld) but current is (withheld)' } }, + // adr0112-ok: D6b persisted audit column previous: { id: 'd-1', code: 'metadata_conflict', note: `expected parent ${PARENT} but current is ${HASH}` }, + // adr0112-ok: D6b persisted audit column result: { id: 'd-1', code: 'metadata_conflict', note: 'expected parent (withheld) but current is (withheld)' }, session: {}, }); diff --git a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts index 43d2b5aaa87..386a02f5484 100644 --- a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts +++ b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts @@ -100,6 +100,7 @@ export function withheldStoredHashTokens(text: string): string { export const METADATA_DECISION_AUDIT_OBJECT = 'sys_metadata_audit'; /** The persisted code of an optimistic-concurrency refusal's decision-audit row. */ +// adr0112-ok: D6b — the persisted audit column's own vocabulary, not a wire code const CONFLICT_NOTE_CODE = 'metadata_conflict'; /** The conflict note's one sentence, as every writer of it has spelled it. */ @@ -404,7 +405,10 @@ export async function migrateStoredMetadataBodyCopies( const copiedObjects = [...STORED_METADATA_BODY_OBJECTS, METADATA_DECISION_AUDIT_OBJECT]; const tables: ReadonlyArray<{ object: string; where: Record }> = [ ...STORED_METADATA_BODY_AUDIT_OBJECTS.map((object) => ({ object, where: { object_name: { $in: copiedObjects } } })), - { object: METADATA_DECISION_AUDIT_OBJECT, where: { code: CONFLICT_NOTE_CODE } }, + // A PREDICATE on the persisted `code` column (ADR-0112 D6b, the audit + // column's own vocabulary), written in operator form: it reads rows by the + // code they carry and stamps none. + { object: METADATA_DECISION_AUDIT_OBJECT, where: { code: { $eq: CONFLICT_NOTE_CODE } } }, ]; for (const { object, where } of tables) { report.byObject[object] = { scanned: 0, rewritten: 0 }; diff --git a/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts index abc63cbbb61..e5a3050ea9b 100644 --- a/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts +++ b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts @@ -35,6 +35,8 @@ const metaSnapshot = () => ({ id: 'm1', name: 'v', type: 'view', scope: 'platfor const historySnapshot = () => ({ id: 'h1', name: 'v', type: 'view', metadata: view('one'), checksum: HASH, previous_checksum: PARENT }); const conflictNote = `expected parent ${PARENT} but current is ${HASH}`; const WITHHELD_NOTE = 'expected parent (withheld) but current is (withheld)'; +/** The decision-audit `code` column's own vocabulary (ADR-0112 D6b), not an error code. */ +const CONFLICT_CODE = 'metadata_conflict'; // adr0112-ok: D6b persisted audit column describe('planAuditRowPatch / planActivityRowPatch — the hash columns leave the copy', () => { it('a sys_metadata create snapshot loses its checksum, and nothing else', () => { @@ -83,22 +85,22 @@ describe('planAuditRowPatch / planActivityRowPatch — the hash columns leave th describe('planDecisionNotePatch — the decision-audit note and its copies', () => { it('withholds both hashes of a conflict note', () => { - const patch = planDecisionNotePatch('sys_metadata_audit', { id: 'd1', code: 'metadata_conflict', note: conflictNote }); + const patch = planDecisionNotePatch('sys_metadata_audit', { id: 'd1', code: CONFLICT_CODE, note: conflictNote }); expect(patch).toEqual({ note: WITHHELD_NOTE }); }); it('keeps a null side as null', () => { - const patch = planDecisionNotePatch('sys_metadata_audit', { id: 'd2', code: 'metadata_conflict', note: `expected parent null but current is ${HASH}` }); + const patch = planDecisionNotePatch('sys_metadata_audit', { id: 'd2', code: CONFLICT_CODE, note: `expected parent null but current is ${HASH}` }); expect(patch).toEqual({ note: 'expected parent null but current is (withheld)' }); }); it('leaves every other note alone, and is idempotent', () => { expect(planDecisionNotePatch('sys_metadata_audit', { id: 'd3', code: 'ok', note: 'restored from version 2' })).toBeNull(); - expect(planDecisionNotePatch('sys_metadata_audit', { id: 'd4', code: 'metadata_conflict', note: WITHHELD_NOTE })).toBeNull(); + expect(planDecisionNotePatch('sys_metadata_audit', { id: 'd4', code: CONFLICT_CODE, note: WITHHELD_NOTE })).toBeNull(); }); it('rewrites the ledger and activity copies of a conflict note', () => { - const snapshot = { id: 'd1', type: 'view', name: 'v', operation: 'save', outcome: 'denied', code: 'metadata_conflict', note: conflictNote }; + const snapshot = { id: 'd1', type: 'view', name: 'v', operation: 'save', outcome: 'denied', code: CONFLICT_CODE, note: conflictNote }; const audit = planDecisionNotePatch('sys_audit_log', { id: 'a5', object_name: 'sys_metadata_audit', record_id: 'd1', new_value: JSON.stringify(snapshot), old_value: null, }); @@ -139,12 +141,12 @@ describe('migrateStoredMetadataBodyCopies — the content-hash copies (driven)', sys_metadata: [{ id: 'm1', type: 'view', name: 'v' }], sys_audit_log: [ { id: 'a1', object_name: 'sys_metadata', record_id: 'm1', new_value: JSON.stringify(metaSnapshot()), old_value: null }, - { id: 'a2', object_name: 'sys_metadata_audit', record_id: 'd1', new_value: JSON.stringify({ id: 'd1', code: 'metadata_conflict', note: conflictNote }), old_value: null }, + { id: 'a2', object_name: 'sys_metadata_audit', record_id: 'd1', new_value: JSON.stringify({ id: 'd1', code: CONFLICT_CODE, note: conflictNote }), old_value: null }, { id: 'a3', object_name: 'file_blob', record_id: 'f1', new_value: JSON.stringify({ id: 'f1', checksum: HASH }), old_value: null }, ], sys_activity: [{ id: 'ac1', object_name: 'sys_metadata_history', record_id: 'h1', metadata: JSON.stringify({ old: null, new: historySnapshot() }) }], sys_metadata_audit: [ - { id: 'd1', code: 'metadata_conflict', note: conflictNote }, + { id: 'd1', code: CONFLICT_CODE, note: conflictNote }, { id: 'd2', code: 'ok', note: 'active' }, ], }); diff --git a/scripts/engine-double-contract.pinned.json b/scripts/engine-double-contract.pinned.json index 5f90bd3b1f8..ebc6f6b26cc 100644 --- a/scripts/engine-double-contract.pinned.json +++ b/scripts/engine-double-contract.pinned.json @@ -486,6 +486,11 @@ "verb": "update", "pinned": 1 }, + { + "file": "packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts", + "verb": "findOne", + "pinned": 2 + }, { "file": "packages/metadata-protocol/src/protocol.data-door-stored-metadata-redaction.test.ts", "verb": "findOne", @@ -1331,6 +1336,21 @@ "verb": "update", "pinned": 1 }, + { + "file": "packages/metadata-protocol/src/protocol.served-content-hash.test.ts", + "verb": "delete", + "pinned": 1 + }, + { + "file": "packages/metadata-protocol/src/protocol.served-content-hash.test.ts", + "verb": "findOne", + "pinned": 1 + }, + { + "file": "packages/metadata-protocol/src/protocol.served-content-hash.test.ts", + "verb": "update", + "pinned": 1 + }, { "file": "packages/metadata-protocol/src/protocol.stored-conversions.test.ts", "verb": "delete", @@ -2491,6 +2511,16 @@ "verb": "update", "pinned": 1 }, + { + "file": "packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts", + "verb": "findOne", + "pinned": 1 + }, + { + "file": "packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts", + "verb": "update", + "pinned": 1 + }, { "file": "packages/plugins/plugin-auth/src/accept-invitation-adopt-membership.test.ts", "verb": "delete", From 978def329a7c1fd539d3255c57a980d1855e6c7b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 15:22:43 +0000 Subject: [PATCH 08/11] test(metadata-protocol): the served-content-hash pin's find double holds the caller's bound (#21207) Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .../src/protocol.served-content-hash.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts index aa6e578ce1f..1d9223ef563 100644 --- a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts +++ b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts @@ -101,10 +101,14 @@ function makeEngine(opts: { provider?: boolean } = {}) { if (table === 'sys_metadata_history') return historyRows.find((h) => matchesWhere(h, o.where)) ?? null; return findRow(o.where)?.row ?? null; }, - async find(table: string, o: { where?: Record } = {}) { - if (table === 'sys_metadata_audit') return auditRows.filter((a) => matchesWhere(a, o.where)); - if (table === 'sys_metadata_history') return historyRows.filter((h) => matchesWhere(h, o.where)); - return Array.from(rows.values()).filter((r) => matchesWhere(r, o.where)); + async find(table: string, o: { where?: Record; limit?: number } = {}) { + const matched = table === 'sys_metadata_audit' + ? auditRows.filter((a) => matchesWhere(a, o.where)) + : table === 'sys_metadata_history' + ? historyRows.filter((h) => matchesWhere(h, o.where)) + : Array.from(rows.values()).filter((r) => matchesWhere(r, o.where)); + // The caller's bound, applied after the filter, by presence. + return typeof o?.limit === 'number' ? matched.slice(0, o.limit) : matched; }, async insert(table: string, data: Record) { nextId += 1; From 1ad5a0099e176ec14919d85ca809964e6c225abe Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 15:48:29 +0000 Subject: [PATCH 09/11] test(metadata-protocol): the served-content-hash pin's where-matcher refuses combinators it does not implement (#21207) Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .../metadata-protocol/src/protocol.served-content-hash.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts index 1d9223ef563..62c5c0fc945 100644 --- a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts +++ b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts @@ -71,6 +71,8 @@ function matchesWhere(r: Record, where: Record if (!(v as Array>).some((c) => matchesWhere(r, c))) return false; continue; } + // Any other combinator is REFUSED, never read as a field name. + if (k.startsWith('$')) throw new Error(`stub engine: combinator '${k}' is not implemented`); if (v === undefined) continue; if (r[k] !== v) return false; } From 7660d811a742201a77de376488741837a8aed310 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 16:45:26 +0000 Subject: [PATCH 10/11] fix(metadata-protocol)!: with no crypto provider, version tokens are keyed under a process-scoped ephemeral key, never empty (#21207) The first cut served an empty version token on a host with no crypto provider. Every save then handed out the same empty token, and a client that sends no pin for an empty token turned every pinned reset into an unpinned one: the optimistic lock failed open. CI's real reset-door pin caught it. The doors now key under the provider when one is registered and, while none is, under 32 random bytes drawn once per process and never written anywhere. A token is always served, differs when the content differs, is never the unkeyed stored hash, and no empty or withheld token equals it. A token held across a restart, or across a provider's registration, is refused once with 409. The no-provider refusal branch is gone, and the batch-publish conformance pin is back to its base bytes: it passes as it was written. Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .changeset/21207-keyed-served-content-hash.md | 6 +- .../src/metadata-redaction.ts | 55 ++++++-- ...ocol.data-door-stored-content-hash.test.ts | 34 +++-- .../src/protocol.served-content-hash.test.ts | 123 ++++++++++++++---- packages/metadata-protocol/src/protocol.ts | 82 ++++++------ .../stored-metadata-body-family.pin.test.ts | 5 +- ...ackage-drafts-response-conformance.test.ts | 12 +- 7 files changed, 206 insertions(+), 111 deletions(-) diff --git a/.changeset/21207-keyed-served-content-hash.md b/.changeset/21207-keyed-served-content-hash.md index a8734ff4b58..ea544f92d63 100644 --- a/.changeset/21207-keyed-served-content-hash.md +++ b/.changeset/21207-keyed-served-content-hash.md @@ -11,14 +11,14 @@ fix(metadata-protocol)!: a metadata body's stored content hash is served and com Clause-②: yes (narrowing) - + **BREAKING**: this narrows what the metadata doors serve and accept for the stored content hash of a metadata body — a hash over the whole stored body, withheld credential material included. Served beside the projected body it let a reader confirm a guess at that material offline; filtered on, it confirmed one online. It ships as `minor` under the launch-window convention for accept-set narrowings. **Three things change for callers and operators.** -1. **A held version token gets one `409 METADATA_CONFLICT`.** Every door that hands out a metadata version token — the save, publish, package-publish and rollback receipts and the history read — now hands out the crypto provider's keyed digest of the stored hash instead of the hash itself, and the save and reset doors compare a token they are sent in that same form. A token a client held from before the upgrade is refused once; take the token from the next read or receipt and retry. A deployment that registers no crypto provider issues no token — the receipts' version is empty and the history read serves none — and refuses a sent token with the same `409`; send the write without one, or register a provider. +1. **A held version token gets one `409 METADATA_CONFLICT`.** Every door that hands out a metadata version token — the save, publish, package-publish and rollback receipts and the history read — now hands out a keyed digest of the stored hash instead of the hash itself, and the save and reset doors compare a token they are sent in that same form. The key is the crypto provider's; a host that registers none keys under a process-scoped ephemeral key instead, so a token is always issued and never empty. A token a client held from before the upgrade is refused once; take the token from the next read or receipt and retry. On a host with no provider the same happens after a restart, and on any host when a provider is first registered. An empty, withheld, raw or stale token is refused with the same `409`; it is never read as "no pin". 2. **Filter, sort and group on the two stored content-hash columns, and on the version history's change note, now answer `400 INVALID_FIELD`** — on the generic data door, the MCP stdio reader and the analytics door, before the engine runs. The change note is included because a draft promotion that stated no message of its own recorded the draft's stored hash in it; the publish door now always states a hash-free message, and a note written before this release is served with the quoted hash in keyed form. A data-door search over the two stored-metadata tables no longer scans those columns or the stored body column, and an explicit search-field list naming one answers the same `400`. Every other column of the two tables is served, filtered, sorted and grouped as before, and every other object is unchanged. 3. **Operators run `os migrate audit-metadata-bodies` once after upgrading, dry run first.** The audit ledger, the activity feed and the metadata decision-audit trail no longer copy the stored hash. The extended command drops it from the copies already written and withholds it in the decision-audit notes and their copies: a dry run by default, `--apply` to rewrite, idempotent. The version history stays the lineage. -**What else changes.** The data door and the MCP stdio reader serve the two hash columns of the stored-metadata tables in keyed form, or omit them with no provider. A `409` conflict refusal carries keyed values or none. The ObjectQL engine gains a read accessor for the registered provider's keyed digest; it is additive. A member's read of these tables is refused as before. +**What else changes.** The data door serves the two hash columns of the stored-metadata tables in keyed form, under the same key as the version tokens. The MCP stdio reader serves them keyed under the crypto provider's key, and omits them on a host with no provider. A `409` conflict refusal carries keyed values or none. The ObjectQL engine gains a read accessor for the registered provider's keyed digest; it is additive. A member's read of these tables is refused as before. diff --git a/packages/metadata-protocol/src/metadata-redaction.ts b/packages/metadata-protocol/src/metadata-redaction.ts index 1c62629fcf2..b4250d48d73 100644 --- a/packages/metadata-protocol/src/metadata-redaction.ts +++ b/packages/metadata-protocol/src/metadata-redaction.ts @@ -66,6 +66,7 @@ * belongs on that list first; that file is `packages/spec`'s to change. */ +import { createHmac, randomBytes } from 'node:crypto'; import { getMetadataTypeRedactor } from '@objectstack/spec/kernel'; import type { MetadataTypeRedactor } from '@objectstack/spec/kernel'; // [#21120] The family-wide stored-metadata-body primitives — the object set, @@ -829,8 +830,8 @@ const QUOTED_STORED_HASH = /(? { const quoted = text.match(QUOTED_STORED_HASH); @@ -843,12 +844,46 @@ export async function serveStoredHashTokens(text: string, digest: StoredHashDige /** The keyed-digest primitive of the registered crypto provider (`ICryptoProvider.keyedDigest`). */ export type StoredHashDigest = (plain: string) => Promise; +/** + * [#21207] The process key {@link ephemeralStoredHashDigest} keys under: 32 + * random bytes, drawn on first use, held only in this module, never written, + * logged or served. + */ +let ephemeralDigestKey: Buffer | undefined; + +/** + * [#21207] The keyed digest the `/meta` doors serve and compare metadata + * version tokens under while NO crypto provider is registered: `hmac-sha256:` + * plus hex, the provider contract's own output shape, under a process-scoped + * ephemeral key. + * + * Why a key and not "serve nothing": a version token is an optimistic lock. + * Serving none hands every caller the same empty token, and a client that + * (rightly) sends no pin for an empty token turns every pinned write into an + * unpinned one, so the lock fails OPEN without a word. A token keyed under a + * secret nobody outside this process holds keeps the three properties the lock + * needs: it differs when the content differs; it is never the unkeyed stored + * hash, so it confirms no guess at withheld material offline; and no empty or + * withheld value ever equals it. + * + * What it costs: the key dies with the process. A token held across a restart, + * or across the moment a host registers a real provider (the doors read the + * provider per use), names no current version and is refused once with + * `409 METADATA_CONFLICT`; the next read or receipt serves the current one. + * Every protocol in one process shares this key, so per-environment protocols + * answer one another's tokens. + */ +export const ephemeralStoredHashDigest: StoredHashDigest = async (plain: string): Promise => { + ephemeralDigestKey ??= randomBytes(32); + return `hmac-sha256:${createHmac('sha256', ephemeralDigestKey).update(plain, 'utf8').digest('hex')}`; +}; + /** * The form a stored content hash is SERVED in: the keyed digest of the stored - * value under the provider's server-held key; `null` when nothing is stored - * (a delete event, a first version's parent); `undefined` — WITHHELD — when no - * provider is registered, or when the stored value is not a string this - * function can judge. ⛔ Never the stored value itself. + * value under a server-held key; `null` when nothing is stored (a delete + * event, a first version's parent); `undefined` (WITHHELD) when the caller + * holds no digest, or when the stored value is not a string this function can + * judge. ⛔ Never the stored value itself. * * A failing digest is not caught: a provider that cannot compute it fails the * read rather than serving what it exists to replace. @@ -865,9 +900,9 @@ export async function servedContentHash( /** * Serve one row of a stored-metadata table with its content-hash columns in * their served form ({@link servedContentHash}): keyed, `null` kept `null`, and - * the column OMITTED when the value is withheld — and, with no crypto provider, - * both columns omitted outright. A row of any other object, and a row carrying - * neither column, is returned by reference. + * the column OMITTED when the value is withheld — and, when the caller holds no + * digest, both columns omitted outright. A row of any other object, and a row + * carrying neither column, is returned by reference. */ export async function serveStoredMetadataHashColumns( object: string, @@ -879,7 +914,7 @@ export async function serveStoredMetadataHashColumns( const out: Record = { ...row }; for (const column of STORED_METADATA_HASH_COLUMNS) { if (!(column in out)) continue; - // No provider: the column is not served at all — a `null` included, so + // No digest: the column is not served at all — a `null` included, so // a reader cannot tell a withheld hash from an absent one either. const served = digest ? await servedContentHash(out[column], digest) : undefined; if (served === undefined) delete out[column]; diff --git a/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts b/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts index a30205f478c..1559095ee71 100644 --- a/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts +++ b/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts @@ -14,7 +14,7 @@ * - SERVED: each column is the crypto provider's keyed digest of the stored * value — neither the stored value nor a recomputation, stable across reads, * moving on any body change including a credential-only one; a `null` stays - * `null`; with no provider both columns are omitted; + * `null`; with no provider the key is the process-scoped ephemeral one; * - EVALUATED: a filter, sort or grouping on either column is refused before * the engine is asked — `INVALID_FIELD` / 400, the shape the body column's * refusals already answer — and a search never scans them (nor the body @@ -156,20 +156,23 @@ describe('[#21207] data door — the content-hash columns are served keyed', () expect(got.record.previous_checksum).toMatch(KEYED); }); - it('no provider: both columns are omitted on every read, the other columns untouched', async () => { + it('no provider: both columns keyed under the process key, never stored, stable across reads', async () => { const { p } = makeProtocol({ provider: false }); const list: any = await p.findData({ object: 'sys_metadata', query: {} }); - for (const row of list.records) { - expect('checksum' in row).toBe(false); - expect(typeof row.name).toBe('string'); + const again: any = await p.findData({ object: 'sys_metadata', query: {} }); + for (const row of SYS_METADATA_ROWS) { + const served = byId(list.records, row.id); + expect(served.checksum).toMatch(KEYED); + expect(served.checksum).not.toBe(row.checksum); + expect(served.checksum).not.toBe(await keyedDigest(row.checksum)); + expect(byId(again.records, row.id).checksum).toBe(served.checksum); + expect(typeof served.name).toBe('string'); } const hist: any = await p.findData({ object: 'sys_metadata_history', query: {} }); - for (const row of hist.records) { - expect('checksum' in row).toBe(false); - expect('previous_checksum' in row).toBe(false); - } + expect(byId(hist.records, 'h_1').previous_checksum).toBeNull(); + for (const row of hist.records) expect(row.checksum).toMatch(KEYED); const got: any = await p.getData({ object: 'sys_metadata', id: 'm_view' }); - expect('checksum' in got.record).toBe(false); + expect(got.record.checksum).toBe(byId(list.records, 'm_view').checksum); expect(got.record.name).toBe('all_tasks'); }); @@ -261,7 +264,7 @@ describe('[#21207] data door — the history change note that quotes a stored ha const QUOTED = hashSpec({ quoted: true }); const NOTE_ROWS = [{ id: 'h_note', type: 'view', name: 'all_tasks', version: 3, operation_type: 'publish', metadata: '{}', checksum: QUOTED, previous_checksum: null, change_note: `publish draft (hash ${QUOTED})` }]; - it('is served with the quote keyed, and withheld with no provider', async () => { + it('is served with the quote keyed, under the provider\'s key or the process key', async () => { const saved = [...HISTORY_ROWS]; HISTORY_ROWS.push(...(NOTE_ROWS as any)); try { @@ -269,9 +272,12 @@ describe('[#21207] data door — the history change note that quotes a stored ha const got: any = await p.getData({ object: 'sys_metadata_history', id: 'h_note' }); expect(got.record.change_note).toBe(`publish draft (hash ${await keyedDigest(QUOTED)})`); const bare = makeProtocol({ provider: false }); - const withheld: any = await bare.p.findData({ object: 'sys_metadata_history', query: {} }); - expect(byId(withheld.records, 'h_note').change_note).toBe('publish draft (hash (withheld))'); - expect(JSON.stringify(withheld.records)).not.toContain(QUOTED); + const keyed: any = await bare.p.findData({ object: 'sys_metadata_history', query: {} }); + const note = byId(keyed.records, 'h_note'); + // The quote is served as the row's own served hash column. + expect(note.change_note).toBe(`publish draft (hash ${note.checksum})`); + expect(note.checksum).toMatch(KEYED); + expect(JSON.stringify(keyed.records)).not.toContain(QUOTED); } finally { HISTORY_ROWS.length = 0; HISTORY_ROWS.push(...saved); diff --git a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts index 62c5c0fc945..2a69d3c06cb 100644 --- a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts +++ b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts @@ -12,8 +12,9 @@ * about withheld credential material offline. Every door that takes a version * token back (the save door's and the reset door's optimistic lock) compares it * in keyed form against the current stored value and hands the STORED value to - * the repository. With no provider registered nothing is served and an inbound - * token is refused: fail closed. + * the repository. With no provider registered the key is a process-scoped + * ephemeral one: tokens are still served, so the optimistic lock never fails + * open on an empty token. * * Pinned per door, as an administrator would read it: * - the served value is neither the stored hash nor a recomputation from the @@ -22,7 +23,8 @@ * the ADR-0112 envelope (`METADATA_CONFLICT` / 409); * - the refusal's text and attributes carry the keyed value or none, and the * decision-audit note it writes carries no hash at all; - * - no provider: nothing served, every inbound token refused. + * - no provider: tokens keyed under the process key, never empty; an empty, + * withheld, raw or stale token is refused on every door. * * The engine is an in-memory double of the stored tables with the repository's * own read and write shapes (the `protocol.lifecycle-audit-rows.test.ts` double, @@ -329,38 +331,101 @@ describe('[#21207] inbound version tokens are compared in keyed form', () => { }); }); -describe('[#21207] no crypto provider: nothing served, every inbound token refused (fail closed)', () => { - it('receipts and history serve no hash; an unconditional save still writes', async () => { +describe('[#21207] no crypto provider: tokens keyed under a process-scoped ephemeral key', () => { + it('receipts and history serve a keyed token: never empty, never stored, distinct per content, stable', async () => { const h = makeEngine({ provider: false }); const p = new ObjectStackProtocolImplementation(h.engine); - const saved: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); - expect(saved.success).toBe(true); - expect(saved.version).toBe(''); - await p.saveMetaItem({ ...ref, item: viewBody('v2') } as any); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const stored1 = activeHash(h); + expect(v1.version).toMatch(KEYED); + expect(v1.version).not.toBe(stored1); + expect(v1.version).not.toBe(hashSpec(viewBody('v1'))); + // Not this file's test key either: the key is the process's own. + expect(v1.version).not.toBe(await keyedDigest(stored1)); - const { events } = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); - expect(events.length).toBeGreaterThanOrEqual(2); - for (const ev of events) { - expect(ev.hash).toBeNull(); - expect(ev.parentHash).toBeNull(); + const v2: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + expect(v2.version).toMatch(KEYED); + expect(v2.version).not.toBe(v1.version); + + const read1 = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + const read2 = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + expect(read1.events.length).toBeGreaterThanOrEqual(2); + for (const ev of read1.events) expect(ev.hash).toMatch(KEYED); + expect(read1.events.map((e) => e.hash)).toContain(v2.version); + expect(JSON.stringify(read2)).toBe(JSON.stringify(read1)); + expectNoStoredHash(JSON.stringify({ v1, v2, read1 }), storedHashes(h)); + }); + + it('the served token is accepted on both doors; a stale, raw, empty or withheld token is refused (METADATA_CONFLICT / 409)', async () => { + const h = makeEngine({ provider: false }); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const v2: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + expect(v2.success).toBe(true); + const raw = activeHash(h); + + for (const token of [v1.version, raw, '', '(withheld)']) { + for (const run of [ + () => p.saveMetaItem({ ...ref, item: viewBody('lost'), parentVersion: token } as any), + () => p.deleteMetaItem({ ...ref, parentVersion: token } as any), + ]) { + const refused = await rejection(run); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + expectNoStoredHash(refused, storedHashes(h)); + } } - expectNoStoredHash(JSON.stringify({ saved, events }), storedHashes(h)); + expect(activeHash(h)).toBe(raw); + + const reset: any = await p.deleteMetaItem({ ...ref, parentVersion: v2.version } as any); + expect(reset.success).toBe(true); + }); + + it('one key per process: a second protocol answers the first one\'s token', async () => { + const h = makeEngine({ provider: false }); + const first = new ObjectStackProtocolImplementation(h.engine); + const second = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await first.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const v2: any = await second.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + expect(v2.success).toBe(true); }); - it('an inbound token — raw or not — is refused on both doors, and the refusal carries no hash', async () => { + it('a provider registered later moves the tokens: the held token is refused once, the next served one is accepted', async () => { const h = makeEngine({ provider: false }); const p = new ObjectStackProtocolImplementation(h.engine); + const before: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + + h.engine.getKeyedDigest = () => keyedDigest; + const refused = await rejection(() => + p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: before.version } as any)); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + + const { events } = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + const current = events.find((e) => e.hash === refused.actualHead); + expect(refused.actualHead).toBe(await keyedDigest(activeHash(h))); + expect(current).toBeDefined(); + const after: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: refused.actualHead } as any); + expect(after.success).toBe(true); + expect(after.version).toBe(await keyedDigest(activeHash(h))); + }); +}); + +describe('[#21207] a sent token is never read as no pin', () => { + it('an empty or withheld token is refused on both doors with a provider registered (METADATA_CONFLICT / 409)', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); const raw = activeHash(h); - - for (const run of [ - () => p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: raw } as any), - () => p.deleteMetaItem({ ...ref, parentVersion: raw } as any), - ]) { - const refused = await rejection(run); - expect(refused.code).toBe('METADATA_CONFLICT'); - expect(refused.status).toBe(409); - expectNoStoredHash(refused, storedHashes(h)); + for (const token of ['', '(withheld)']) { + for (const run of [ + () => p.saveMetaItem({ ...ref, item: viewBody('lost'), parentVersion: token } as any), + () => p.deleteMetaItem({ ...ref, parentVersion: token } as any), + ]) { + const refused = await rejection(run); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + } } expect(activeHash(h)).toBe(raw); }); @@ -377,11 +442,11 @@ describe('[#21207] a change note that quotes a stored hash', () => { for (const note of notes) expect(note).not.toMatch(SHA256); }); - it('the history read serves a stored note\'s quoted hash keyed, and withheld with no provider', async () => { + it('the history read serves a stored note\'s quoted hash keyed, under the provider\'s key or the process key', async () => { for (const provider of [true, false]) { const h = makeEngine({ provider }); const p = new ObjectStackProtocolImplementation(h.engine); - await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const saved: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); const stored = activeHash(h); // A row written before the publish door stated its own message. for (const row of h.historyRows) row.change_note = `publish draft (hash ${stored})`; @@ -389,7 +454,9 @@ describe('[#21207] a change note that quotes a stored hash', () => { const { events } = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); expect(events.length).toBeGreaterThan(0); for (const ev of events) { - expect(ev.message).toBe(provider ? `publish draft (hash ${await keyedDigest(stored)})` : 'publish draft (hash (withheld))'); + // The quote is served as the very token the receipt served. + expect(ev.message).toBe(`publish draft (hash ${saved.version})`); + if (provider) expect(saved.version).toBe(await keyedDigest(stored)); } expectNoStoredHash(JSON.stringify(events), storedHashes(h)); } diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index a3a912264bf..d880da1ca5a 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -194,6 +194,7 @@ import { storedMetadataBodyProjection, // [#21207] The stored content hash of the same rows: served keyed, never // evaluated — see `STORED_METADATA_HASH_COLUMNS`. + ephemeralStoredHashDigest, isStoredMetadataBodyObject, servedContentHash, serveStoredHashTokens, @@ -2621,21 +2622,17 @@ function declaresClientRefusal(err: unknown): boolean { * A `ConflictError`, so each door's existing conflict branch — the 409 * `METADATA_CONFLICT` and its decision-audit row — handles it unchanged; told * apart from the repository's own race conflict because its `expectedParent` - * is the CALLER's token rather than a stored hash, and `unverifiable` marks the - * deployment with no crypto provider, where no token can be checked at all. - * The message is replaced: the base class prints both values, and one of them - * is the stored hash. + * is the CALLER's token rather than a stored hash. The message is replaced: + * the base class prints both values, and one of them is the stored hash. */ class InboundVersionConflictError extends ConflictError { constructor( ref: { org: string; type: string; name: string }, token: string, currentStored: string | null, - readonly unverifiable: boolean, ) { super(ref as ConstructorParameters[0], token, currentStored); - this.message = `Conflict on ${ref.type}/${ref.name}: the version token sent ` - + (unverifiable ? 'cannot be checked (no crypto provider is registered)' : 'is not the current version'); + this.message = `Conflict on ${ref.type}/${ref.name}: the version token sent is not the current version`; } } @@ -12004,7 +12001,7 @@ export class ObjectStackProtocolImplementation implements // [#21207] …and its stored CONTENT HASH (`checksum`, `previous_checksum`) // is served in keyed form — never the stored value, which beside the // projected body confirms a guess at the withheld material offline — - // and omitted when no crypto provider is registered. + // under the provider's key, or the process-scoped ephemeral one. const records = await serveStoredMetadataHashColumnRows( request.object, redactStoredMetadataRows( @@ -12138,7 +12135,7 @@ export class ObjectStackProtocolImplementation implements object: request.object, id: request.id, // [#21207] Same served form as the list path: the content-hash - // columns keyed, or omitted with no crypto provider. + // columns keyed. record: await serveStoredMetadataHashColumns( request.object, redactStoredMetadataRow(request.object, result, { dropType: bodyProjection.addedType }), @@ -15790,32 +15787,37 @@ export class ObjectStackProtocolImplementation implements // producers and the parent links are untouched — but no door hands it out. // It is a hash over the WHOLE stored body, withheld credential material // included, so beside the projected body it confirms a guess at that - // material offline. Every door that serves it serves the crypto provider's - // keyed digest of it; every door that takes a version token back compares - // the token in that same form and hands the STORED value to the repository; - // with no provider registered nothing is served and every token is refused. + // material offline. Every door that serves it serves a keyed digest of it + // (the crypto provider's, or with none registered a process-scoped + // ephemeral key's); every door that takes a version token back compares the + // token in that same form and hands the STORED value to the repository. /** - * The registered crypto provider's keyed digest, read from the engine at - * the moment of use — a host registers the provider AFTER the kernel starts, - * so a value read once at construction would answer "none" for good. - * `undefined` when none is registered (or the host engine has no such - * accessor, which is the same fact): the doors then withhold and refuse. + * The keyed digest the doors serve and compare under: the registered crypto + * provider's, read from the engine at the moment of use — a host registers + * the provider AFTER the kernel starts, so a value read once at + * construction would answer "none" for good — and, while none is registered + * (or the host engine has no such accessor), {@link ephemeralStoredHashDigest}. + * + * ⛔ Never `undefined`: a door with no key would serve no token, every + * caller would then hold the same empty one, and a client that sends no pin + * for an empty token would turn every pinned write into an unpinned one — + * the optimistic lock failing OPEN. */ - private storedHashDigest(): StoredHashDigest | undefined { + private storedHashDigest(): StoredHashDigest { const accessor = this.engine?.getKeyedDigest; - return typeof accessor === 'function' ? accessor.call(this.engine) : undefined; + const provider: StoredHashDigest | undefined = + typeof accessor === 'function' ? accessor.call(this.engine) : undefined; + return provider ?? ephemeralStoredHashDigest; } /** * A write receipt's `version` — the version token a caller sends back as * `If-Match` — for a write whose stored content hash is `stored`: its keyed - * digest. With no crypto provider the deployment issues no token, and the - * receipt's required `version` carries the empty string: a value that names - * nothing and that every inbound comparison refuses. + * digest ({@link storedHashDigest}). Never empty, never the stored value. */ private async receiptVersion(stored: string): Promise { - return (await servedContentHash(stored, this.storedHashDigest())) ?? ''; + return this.storedHashDigest()(stored); } /** @@ -15824,9 +15826,10 @@ export class ObjectStackProtocolImplementation implements * keyed digest of that head, and the stored value is what the repository's * own optimistic lock then compares. `null` keeps its meaning ("expect no * row") and carries no hash. Anything else — the raw stored hash a pre-keying - * client still holds, a stale token, a token sent to a deployment with no - * crypto provider — is an {@link InboundVersionConflictError}, which the - * door's conflict branch answers 409. + * client still holds, a stale token, an empty or withheld token, a token + * keyed before a restart or before a provider was registered — is an + * {@link InboundVersionConflictError}, which the door's conflict branch + * answers 409. ⛔ A sent token is never read as "no pin". */ private async storedParentForToken( ref: { org: string; type: string; name: string }, @@ -15834,10 +15837,10 @@ export class ObjectStackProtocolImplementation implements currentStored: string | null, ): Promise { if (token === null) return null; - const digest = this.storedHashDigest(); - if (!digest) throw new InboundVersionConflictError(ref, token, currentStored, true); - if (currentStored !== null && (await digest(currentStored)) === token) return currentStored; - throw new InboundVersionConflictError(ref, token, currentStored, false); + if (currentStored !== null && token !== '' && (await this.storedHashDigest()(currentStored)) === token) { + return currentStored; + } + throw new InboundVersionConflictError(ref, token, currentStored); } /** @@ -15848,8 +15851,7 @@ export class ObjectStackProtocolImplementation implements * * - a repository race (the stored head moved between the door's read and * its write): `Expected parent X but current is Y`, both keyed; - * - a caller's token naming no current head: the keyed current head; - * - no crypto provider: no value, and the remedy. + * - a caller's token naming no current head: the keyed current head. * * A side with no served form is `(withheld)`; an absent side is `null`. */ @@ -15857,13 +15859,6 @@ export class ObjectStackProtocolImplementation implements const conflict: any = new Error(subject); conflict.code = 'METADATA_CONFLICT'; conflict.status = 409; - if (err instanceof InboundVersionConflictError && err.unverifiable) { - conflict.message = `${subject}: the version token sent cannot be checked, so the write was not run. ` - + 'This deployment registers no crypto provider, so it issues no version tokens and compares none. ' - + 'Send the write without a version token (no If-Match) to write unconditionally, or register a ' - + 'crypto provider (engine.setCryptoProvider) so version tokens are issued and checked.'; - return conflict; - } const digest = this.storedHashDigest(); const show = (served: string | null | undefined) => (served === undefined ? '(withheld)' : served ?? 'null'); const current = await servedContentHash(err.actualHead, digest); @@ -18678,8 +18673,8 @@ export class ObjectStackProtocolImplementation implements item, mode: state === 'draft' ? 'draft' : 'publish', // [#21207] The STORED hash this pass read itself — the - // in-process spelling, never compared in keyed form (a - // deployment with no crypto provider still migrates). + // in-process spelling, never compared in keyed form (the + // pass already holds the stored value; nothing to key). storedParentVersion: row.checksum ?? null, packageId, force: true, @@ -18798,8 +18793,7 @@ export class ObjectStackProtocolImplementation implements if (request.limit !== undefined) opts.limit = request.limit; // [#21207] Each event's hash and parent hash are served in keyed form — // the same value the write receipts hand out, so an event still names - // the token a caller holds — and `null` with no crypto provider (a - // delete event's own `null` is kept either way). + // the token a caller holds; a delete event's own `null` is kept. // The event's message is the row's change note, which can QUOTE a stored // hash (`publish draft (hash …)` on rows written before the publish door // stated its own message) — each quote is served the same way. diff --git a/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts b/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts index 7647c50e02e..a4769ff2d97 100644 --- a/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts +++ b/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts @@ -42,8 +42,9 @@ * [#21207] Exit two — the same rows' stored CONTENT HASH (`checksum`, and the * history table's `previous_checksum`), a hash over the whole stored body, * withheld credential material included. Every surface that SERVES it serves - * the crypto provider's keyed digest (`keyed`), every surface that takes a - * version token back compares it keyed, every COPY drops it (`withheld`), and + * a keyed digest (`keyed`: the crypto provider's, or a process-scoped ephemeral + * key's while none is registered), every surface that takes a version token + * back compares it keyed, every COPY drops it (`withheld`), and * every EVALUATE shape refuses — enumerated as rows below, each with its pin, * plus a third tooth: the hash columns the object definitions declare are * exactly `STORED_METADATA_HASH_COLUMNS`, so a new hash-like column fails here diff --git a/packages/objectql/src/publish-package-drafts-response-conformance.test.ts b/packages/objectql/src/publish-package-drafts-response-conformance.test.ts index 6a753b8549c..63f687725a1 100644 --- a/packages/objectql/src/publish-package-drafts-response-conformance.test.ts +++ b/packages/objectql/src/publish-package-drafts-response-conformance.test.ts @@ -28,7 +28,6 @@ * upgrade only when a consumer needs a field) — so the cases here assert it is * carried through unstripped, never its inner shape. */ -import { createHash } from 'node:crypto'; import { describe, it, expect } from 'vitest'; import type { ServiceObject } from '@objectstack/spec/data'; import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; @@ -139,12 +138,6 @@ async function makeProtocol() { // only into a package the registry holds (#20863). Its manifest declares // no namespace, so the ADR-0028 prefix pre-flight is unchanged. engine.registry.installPackage({ id: PKG, name: 'Edu', version: '1.0.0' } as never); - // [#21207] The host's composition-root step (`os serve`, the verify harness): - // a crypto provider. A version token is the provider's keyed digest of the - // stored content hash, so without one the receipts issue none. - engine.setCryptoProvider({ - keyedDigest: async (plain: string) => `hmac-sha256:${createHash('sha256').update(`conformance|${plain}`).digest('hex')}`, - } as never); return new ObjectStackProtocolImplementation(engine); } @@ -223,11 +216,10 @@ describe('publishPackageDrafts response conforms to PublishPackageDraftsResponse expect(parsed.publishedCount).toBe(2); expect(parsed.failedCount).toBe(0); expect(parsed.failed).toEqual([]); - // Every element carries the ADR-0008 OCC token, unstripped — [#21207] - // in keyed form, never the stored content hash. + // Every element carries the ADR-0008 OCC token, unstripped. for (const el of parsed.published) { expect(typeof el.version).toBe('string'); - expect(el.version).toMatch(/^hmac-sha256:[0-9a-f]{64}$/); + expect(el.version.length).toBeGreaterThan(0); } expect(parsed.published.map((e) => e.name).sort()).toEqual(['cases', 'leads']); }); From b53bea8da595bab212d688c23e99b943d850db90 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 18:51:43 +0000 Subject: [PATCH 11/11] test(cli): the absent-database audit pin counts every audited table, the decision-audit trail included (#21207) #21432 (#21391) pinned that a dry run of audit-metadata-bodies on an absent database exits 1 and counts every table it audits as unread. It was written while the audit read two tables. This branch widens the audit to the decision-audit trail, so the merged tree reads three. Both intents are stacked: failures equals the whole audited set, byObject names exactly those tables, scanned stays 0, exit 1. The set is stated literally, so a later widening turns the case red rather than passing on a stale count. Claude-Session: https://claude.ai/code/session_01VvcEokUG1tvVxkceYfR5XB Co-authored-by: Claude --- .../migrate/preview-read-only.integration.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/commands/migrate/preview-read-only.integration.test.ts b/packages/cli/src/commands/migrate/preview-read-only.integration.test.ts index 637d8da37b4..8a0fd369d02 100644 --- a/packages/cli/src/commands/migrate/preview-read-only.integration.test.ts +++ b/packages/cli/src/commands/migrate/preview-read-only.integration.test.ts @@ -493,16 +493,22 @@ for (const cell of DIALECT_CELLS) { expect(payload.error).toContain("'sys_metadata'"); }, cell.timeout); - it('audit-metadata-bodies without --apply on a database that does not exist exits 1 with both tables counted unread', async () => { + // [#21207] The audit reads a third table: the decision-audit trail, whose + // conflict notes named stored content hashes. The #21391 intent is + // unchanged — EVERY audited table is counted unread — so the expected set + // is the whole audited set, stated literally: a widening that is not + // carried here turns this case red instead of passing on a stale count. + it('audit-metadata-bodies without --apply on a database that does not exist exits 1 with every audited table counted unread', async () => { const absent = join(fixture!.dir, 'data', 'never-started.db'); const { payload, exitCode } = await runJson(auditBodies, ['--database-url', `file:${absent}`]); + const audited = ['sys_activity', 'sys_audit_log', 'sys_metadata_audit']; expect(exitCode).toBe(1); expect(payload.apply).toBe(false); // `failures` counts the tables whose rows were NOT examined. - expect(payload.report.failures).toBe(2); + expect(payload.report.failures).toBe(audited.length); expect(payload.report.scanned).toBe(0); - expect(Object.keys(payload.report.byObject).sort()).toEqual(['sys_activity', 'sys_audit_log']); + expect(Object.keys(payload.report.byObject).sort()).toEqual(audited); }, cell.timeout); } });