diff --git a/.changeset/21207-keyed-served-content-hash.md b/.changeset/21207-keyed-served-content-hash.md new file mode 100644 index 00000000000..ea544f92d63 --- /dev/null +++ b/.changeset/21207-keyed-served-content-hash.md @@ -0,0 +1,24 @@ +--- +'@objectstack/metadata-protocol': minor +'@objectstack/objectql': minor +'@objectstack/mcp': minor +'@objectstack/plugin-audit': minor +'@objectstack/service-analytics': minor +'@objectstack/cli': minor +--- + +fix(metadata-protocol)!: a metadata body's stored content hash is served and compared only in keyed form, never copied, and never evaluated (#21207) + +Clause-②: yes (narrowing) + + + +**BREAKING**: this narrows what the metadata doors serve and accept for the stored content hash of a metadata body — a hash over the whole stored body, withheld credential material included. Served beside the projected body it let a reader confirm a guess at that material offline; filtered on, it confirmed one online. It ships as `minor` under the launch-window convention for accept-set narrowings. + +**Three things change for callers and operators.** + +1. **A held version token gets one `409 METADATA_CONFLICT`.** Every door that hands out a metadata version token — the save, publish, package-publish and rollback receipts and the history read — now hands out a keyed digest of the stored hash instead of the hash itself, and the save and reset doors compare a token they are sent in that same form. The key is the crypto provider's; a host that registers none keys under a process-scoped ephemeral key instead, so a token is always issued and never empty. A token a client held from before the upgrade is refused once; take the token from the next read or receipt and retry. On a host with no provider the same happens after a restart, and on any host when a provider is first registered. An empty, withheld, raw or stale token is refused with the same `409`; it is never read as "no pin". +2. **Filter, sort and group on the two stored content-hash columns, and on the version history's change note, now answer `400 INVALID_FIELD`** — on the generic data door, the MCP stdio reader and the analytics door, before the engine runs. The change note is included because a draft promotion that stated no message of its own recorded the draft's stored hash in it; the publish door now always states a hash-free message, and a note written before this release is served with the quoted hash in keyed form. A data-door search over the two stored-metadata tables no longer scans those columns or the stored body column, and an explicit search-field list naming one answers the same `400`. Every other column of the two tables is served, filtered, sorted and grouped as before, and every other object is unchanged. +3. **Operators run `os migrate audit-metadata-bodies` once after upgrading, dry run first.** The audit ledger, the activity feed and the metadata decision-audit trail no longer copy the stored hash. The extended command drops it from the copies already written and withholds it in the decision-audit notes and their copies: a dry run by default, `--apply` to rewrite, idempotent. The version history stays the lineage. + +**What else changes.** The data door serves the two hash columns of the stored-metadata tables in keyed form, under the same key as the version tokens. The MCP stdio reader serves them keyed under the crypto provider's key, and omits them on a host with no provider. A `409` conflict refusal carries keyed values or none. The ObjectQL engine gains a read accessor for the registered provider's keyed digest; it is additive. A member's read of these tables is refused as before. diff --git a/packages/cli/src/commands/migrate/audit-metadata-bodies.ts b/packages/cli/src/commands/migrate/audit-metadata-bodies.ts index f19931b6a7e..40b264029f4 100644 --- a/packages/cli/src/commands/migrate/audit-metadata-bodies.ts +++ b/packages/cli/src/commands/migrate/audit-metadata-bodies.ts @@ -44,16 +44,26 @@ async function confirm(question: string): Promise { * NEW writes; rows copied before the fix keep their cleartext. This command * rewrites them, projecting each copied body through the SAME redactor. * + * [#21207] The same copies also carried the copied row's stored CONTENT HASH + * (`checksum`, and the history row's `previous_checksum`) — a hash over the whole + * stored body, withheld credential material included — and the decision-audit + * note of a refused optimistic-lock write (`sys_metadata_audit`, and its ledger + * and activity copies) named both hashes. The writers no longer copy either; + * this command drops the hash columns from the copies already written and + * withholds the hashes in those notes, in the same pass. Operators run it once + * after upgrading, dry run first. + * * Dry run by default (writes nothing), `--apply` to rewrite. Idempotent: a - * second run finds nothing — a redacted copy has no credential left — so - * re-running and reading a clean report is the verification. No `sys_migration` - * flag is recorded: nothing gates irreversible behaviour on this rewrite (the - * posture `os migrate summary-nulls` takes). + * second run finds nothing — a redacted copy has no credential and no hash + * left — so re-running and reading a clean report is the verification. No + * `sys_migration` flag is recorded: nothing gates irreversible behaviour on this + * rewrite (the posture `os migrate summary-nulls` takes). */ export default class MigrateAuditMetadataBodies extends Command { static override description = - 'Rewrite at-rest cleartext metadata-body copies the audit writer left in sys_audit_log / sys_activity, ' + - 'projecting each copied body through the shared credential redactor. Dry run by default; --apply writes.'; + 'Rewrite the at-rest copies the audit writer left in sys_audit_log / sys_activity: project each copied ' + + 'metadata body through the shared credential redactor and drop its stored content hash; withhold the hashes ' + + 'a conflict note in sys_metadata_audit (and its copies) names. Dry run by default; --apply writes.'; static override examples = [ '$ os migrate audit-metadata-bodies', @@ -117,12 +127,12 @@ export default class MigrateAuditMetadataBodies extends Command { this.exit(1); return; } - printWarning('Apply mode rewrites audit/activity rows. Re-run with --yes to confirm, or run without --apply to preview.'); + printWarning('Apply mode rewrites audit/activity/decision rows. Re-run with --yes to confirm, or run without --apply to preview.'); this.exit(1); return; } const ok = await confirm( - chalk.bold('\nRewrite every audit/activity row carrying a stored metadata body on this database? [y/N] '), + chalk.bold('\nRewrite every audit/activity/decision row carrying a stored metadata body or content hash on this database? [y/N] '), ); if (!ok) { printInfo('Aborted — no changes made.'); @@ -188,14 +198,14 @@ export default class MigrateAuditMetadataBodies extends Command { printError(`${report.failures} row(s) could not be rewritten — re-run to finish them.`); } else if (apply && report.rewritten > 0) { printSuccess( - `Rewrote ${report.rewritten} audit/activity row(s). Re-run any time — it only revisits rows still carrying a body.`, + `Rewrote ${report.rewritten} audit/activity/decision row(s). Re-run any time — it only revisits rows still carrying a body or a hash.`, ); } else if (apply) { - printSuccess('Nothing to rewrite — no audit/activity row carries a stored metadata body.'); + printSuccess('Nothing to rewrite — no audit/activity/decision row carries a stored metadata body or content hash.'); } else if (report.rewritten > 0) { printInfo(`Dry run only — ${report.rewritten} row(s) would be rewritten. Re-run with --apply.`); } else { - printSuccess('Nothing to rewrite — no audit/activity row carries a stored metadata body.'); + printSuccess('Nothing to rewrite — no audit/activity/decision row carries a stored metadata body or content hash.'); } console.log(chalk.dim(` ${timer.display()}`)); console.log(''); diff --git a/packages/cli/src/commands/migrate/preview-read-only.integration.test.ts b/packages/cli/src/commands/migrate/preview-read-only.integration.test.ts index 637d8da37b4..8a0fd369d02 100644 --- a/packages/cli/src/commands/migrate/preview-read-only.integration.test.ts +++ b/packages/cli/src/commands/migrate/preview-read-only.integration.test.ts @@ -493,16 +493,22 @@ for (const cell of DIALECT_CELLS) { expect(payload.error).toContain("'sys_metadata'"); }, cell.timeout); - it('audit-metadata-bodies without --apply on a database that does not exist exits 1 with both tables counted unread', async () => { + // [#21207] The audit reads a third table: the decision-audit trail, whose + // conflict notes named stored content hashes. The #21391 intent is + // unchanged — EVERY audited table is counted unread — so the expected set + // is the whole audited set, stated literally: a widening that is not + // carried here turns this case red instead of passing on a stale count. + it('audit-metadata-bodies without --apply on a database that does not exist exits 1 with every audited table counted unread', async () => { const absent = join(fixture!.dir, 'data', 'never-started.db'); const { payload, exitCode } = await runJson(auditBodies, ['--database-url', `file:${absent}`]); + const audited = ['sys_activity', 'sys_audit_log', 'sys_metadata_audit']; expect(exitCode).toBe(1); expect(payload.apply).toBe(false); // `failures` counts the tables whose rows were NOT examined. - expect(payload.report.failures).toBe(2); + expect(payload.report.failures).toBe(audited.length); expect(payload.report.scanned).toBe(0); - expect(Object.keys(payload.report.byObject).sort()).toEqual(['sys_activity', 'sys_audit_log']); + expect(Object.keys(payload.report.byObject).sort()).toEqual(audited); }, cell.timeout); } }); diff --git a/packages/mcp/src/plugin.ts b/packages/mcp/src/plugin.ts index 0c099302d78..0092d820396 100644 --- a/packages/mcp/src/plugin.ts +++ b/packages/mcp/src/plugin.ts @@ -29,7 +29,9 @@ import { createStdioDataBridge, enforceApiExposure, GATED_ACTIONS, + serveStoredMetadataHashes, serveStoredMetadataRow, + type StoredHashDigest, } from './stdio-data-bridge.js'; import type { McpDataBridge } from './mcp-http-tools.js'; import { CONNECT_AGENT_UI_BUNDLE } from './connect-ui.js'; @@ -377,7 +379,14 @@ export class MCPServerPlugin implements Plugin { let dataBridge: McpDataBridge | undefined; if (shouldStart) { const apiKey = readEnvWithDeprecation('OS_MCP_STDIO_API_KEY', [], { silent: true }); - let ql: (IDataEngine & { find: (object: string, opts: unknown) => Promise }) | undefined; + let ql: + | (IDataEngine & { + find: (object: string, opts: unknown) => Promise; + // [#21207] The engine's keyed-digest accessor (objectql), probed + // per call: an engine without it serves no content hash. + getKeyedDigest?: () => StoredHashDigest | undefined; + }) + | undefined; try { ql = ctx.getService('objectql'); } catch { @@ -546,6 +555,10 @@ export class MCPServerPlugin implements Plugin { // wall that changed mid-session must take effect on the next call rather // than at the next process restart. See `resolveStdioTenancyPosture` for // why this is not hoisted next to the localization memo. + // [#21207] The crypto provider's keyed digest, read at each use — the + // host registers the provider after the kernel starts. + const storedHashDigest = (): StoredHashDigest | undefined => + typeof scopedQl.getKeyedDigest === 'function' ? scopedQl.getKeyedDigest() : undefined; const resolvePrincipal = async (): Promise => { const ec = await resolveStdioExecutionContext( scopedQl, @@ -561,6 +574,7 @@ export class MCPServerPlugin implements Plugin { engine: scopedQl, metadataService, resolvePrincipal, + keyedDigest: storedHashDigest, }); } else { // Functional degradation, said once and naming the remedy: two of the @@ -611,7 +625,12 @@ export class MCPServerPlugin implements Plugin { // `sys_metadata_history` row's body reaches this resource as its type's // read projection, never as the stored bytes — so the tool and the // resource cannot disagree about what a stored credential is. - return serveStoredMetadataRow(objectName, (row ?? null) as Record | null); + // [#21207] …and its stored content hash keyed, or not served at all. + return serveStoredMetadataHashes( + objectName, + serveStoredMetadataRow(objectName, (row ?? null) as Record | null), + storedHashDigest(), + ); }; ctx.logger.info( `[MCP] stdio transport principal-bound to OS_MCP_STDIO_API_KEY identity ${initial.userId} (RLS/FLS/tenant applied)`, diff --git a/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts b/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts new file mode 100644 index 00000000000..e956f69faf5 --- /dev/null +++ b/packages/mcp/src/stdio-data-bridge.stored-content-hash.test.ts @@ -0,0 +1,329 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21207] Exit two at the MCP stdio transport's engine-only reader: the two + * stored content-hash columns of `sys_metadata` / `sys_metadata_history` + * (`checksum`, and the history table's `previous_checksum`). + * + * The engine returns them as stored — a hash over the WHOLE stored body, + * withheld credential material included — so an administrator's key read them + * raw beside the projected body: an offline verifier for a guess at the + * withheld material. Per the maintainer's ruling, this transport now serves + * each as the crypto provider's keyed digest of the stored value (omitted when + * no provider is registered), on every read path — the bridge's query and get + * and the ADR-0101 record resource — and refuses a filter, sort or grouping on + * either column before the engine is asked (`INVALID_FIELD` / 400, the shape + * its body-column refusal already answers). A member stays refused by the + * engine, unchanged. + */ + +import { createHmac } from 'node:crypto'; +import { PassThrough } from 'node:stream'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import { SysMetadataObject, SysMetadataHistoryObject } from '@objectstack/metadata-core'; +import type { ExecutionContext } from '@objectstack/spec/kernel'; +import type { IDataEngine, IMetadataService } from '@objectstack/spec/contracts'; +import type { MCPServerRuntime } from './mcp-server-runtime.js'; +import { MCPServerPlugin } from './plugin.js'; +import { createStdioDataBridge } from './stdio-data-bridge.js'; + +const keyedDigest = async (plain: string): Promise => + `hmac-sha256:${createHmac('sha256', 'mcp-test-key').update(plain, 'utf8').digest('hex')}`; +const KEYED = /^hmac-sha256:[0-9a-f]{64}$/; + +const HASH_M = `sha256:${'1'.repeat(64)}`; +const HASH_H = `sha256:${'2'.repeat(64)}`; +const PARENT_H = `sha256:${'3'.repeat(64)}`; +const STORED = [HASH_M, HASH_H, PARENT_H]; + +const TABLES: Record>> = { + sys_metadata: [{ id: 'm1', type: 'view', name: 'v', metadata: '{"name":"v"}', checksum: HASH_M, state: 'active' }], + sys_metadata_history: [ + { id: 'h1', type: 'view', name: 'v', metadata: '{"name":"v"}', checksum: HASH_H, previous_checksum: PARENT_H }, + { id: 'h0', type: 'view', name: 'v', metadata: '{"name":"v"}', checksum: PARENT_H, previous_checksum: null }, + ], + file_blob: [{ id: 'f1', name: 'blob', checksum: HASH_M }], +}; + +const ADMIN = 'usr_admin'; +const MEMBER = 'usr_member'; + +function matches(row: Record, where: unknown): boolean { + for (const [key, cond] of Object.entries((where ?? {}) as Record)) { + if (key.startsWith('$') || (cond !== null && typeof cond === 'object')) { + throw new Error(`fixture where-matcher: unsupported shape on '${key}'`); + } + if (row[key] !== cond) return false; + } + return true; +} + +function fakeEngine(opts: { owner?: string; provider?: boolean } = {}) { + const owner = opts.owner ?? ADMIN; + const engine: Record = { + find: vi.fn(async (object: string, query?: any, options?: any) => { + const context = options?.context ?? query?.context; + if (object === 'sys_api_key') return [{ id: 'k1', user_id: owner, revoked: false }]; + if (object.startsWith('sys_metadata') && context?.userId !== ADMIN) { + throw Object.assign(new Error(`Permission denied: cannot read '${object}'`), { code: 'PERMISSION_DENIED', status: 403 }); + } + const rows = (TABLES[object] ?? []).filter((row) => matches(row, query?.where)); + return typeof query?.limit === 'number' ? rows.slice(0, query.limit).map((r) => ({ ...r })) : rows.map((r) => ({ ...r })); + }), + aggregate: vi.fn(async () => [] as unknown[]), + }; + if (opts.provider !== false) engine.getKeyedDigest = () => keyedDigest; + return engine; +} + +const DEFS: Record = { + sys_metadata: SysMetadataObject, + sys_metadata_history: SysMetadataHistoryObject, + file_blob: { name: 'file_blob', fields: { name: { type: 'text' }, checksum: { type: 'text' } } }, +}; + +function fakeMetadata() { + return { + listObjects: vi.fn(async () => Object.values(DEFS)), + getObject: vi.fn(async (name: string) => DEFS[name] ?? null), + get: vi.fn(async () => null), + list: vi.fn(async () => []), + exists: vi.fn(async () => true), + getRegisteredTypes: vi.fn(async () => ['object']), + register: vi.fn(), + unregister: vi.fn(), + }; +} + +function bridgeAs(userId: string, opts: { provider?: boolean } = {}) { + const engine = fakeEngine(opts); + const bridge = createStdioDataBridge({ + engine: engine as unknown as IDataEngine, + metadataService: fakeMetadata() as unknown as IMetadataService, + resolvePrincipal: async () => ({ userId, isSystem: false }) as unknown as ExecutionContext, + keyedDigest: () => (engine as { getKeyedDigest?: () => typeof keyedDigest }).getKeyedDigest?.(), + }); + return { bridge, engine: engine as any }; +} + +const noStoredHash = (value: unknown) => { + const text = JSON.stringify(value); + for (const s of STORED) expect(text).not.toContain(s); +}; + +describe('[#21207] stdio reader: an administrator is served the keyed hash, never the stored one', () => { + it('query: both columns keyed on both tables, stable across reads, a null parent stays null', async () => { + const { bridge } = bridgeAs(ADMIN); + const m1 = (await bridge.query('sys_metadata', {})) as { records: Array> }; + const m2 = (await bridge.query('sys_metadata', {})) as { records: Array> }; + expect(m1.records[0]!.checksum).toMatch(KEYED); + expect(m1.records[0]!.checksum).toBe(await keyedDigest(HASH_M)); + expect(m2.records[0]!.checksum).toBe(m1.records[0]!.checksum); + noStoredHash(m1); + + const h = (await bridge.query('sys_metadata_history', {})) as { records: Array> }; + const h1 = h.records.find((r) => r.id === 'h1')!; + expect(h1.checksum).toMatch(KEYED); + expect(h1.previous_checksum).toMatch(KEYED); + expect(h.records.find((r) => r.id === 'h0')!.previous_checksum).toBeNull(); + noStoredHash(h); + }); + + it('get: keyed, equal to the query', async () => { + const { bridge } = bridgeAs(ADMIN); + const row = (await bridge.get('sys_metadata', 'm1')) as Record; + expect(row.checksum).toBe(await keyedDigest(HASH_M)); + noStoredHash(row); + }); + + it('no provider: both columns are omitted', async () => { + const { bridge } = bridgeAs(ADMIN, { provider: false }); + const h = (await bridge.query('sys_metadata_history', {})) as { records: Array> }; + for (const r of h.records) { + expect('checksum' in r).toBe(false); + expect('previous_checksum' in r).toBe(false); + expect(r.name).toBe('v'); + } + const row = (await bridge.get('sys_metadata', 'm1')) as Record; + expect('checksum' in row).toBe(false); + }); + + it('control: another object keeps its own checksum column, raw', async () => { + const { bridge } = bridgeAs(ADMIN); + const res = (await bridge.query('file_blob', {})) as { records: Array> }; + expect(res.records[0]!.checksum).toBe(HASH_M); + }); +}); + +describe('[#21207] stdio reader: evaluating a content-hash column is refused (INVALID_FIELD / 400)', () => { + const CASES: Array<[string, string, (b: ReturnType['bridge']) => Promise, string]> = [ + ['query: filter', 'filter', (b) => b.query('sys_metadata', { where: { checksum: HASH_M } }), 'checksum'], + ['query: filter on the parent hash', 'filter', (b) => b.query('sys_metadata_history', { where: { previous_checksum: PARENT_H } }), 'previous_checksum'], + ['query: sort', 'sort', (b) => b.query('sys_metadata', { orderBy: [{ field: 'checksum', order: 'asc' }] }), 'checksum'], + ['aggregate: group', 'groupBy', (b) => b.aggregate!('sys_metadata', { groupBy: ['checksum'], aggregations: [{ function: 'count', alias: 'n' }] }), 'checksum'], + ['aggregate: group (object form)', 'groupBy', (b) => b.aggregate!('sys_metadata_history', { groupBy: [{ field: 'previous_checksum' }] as any, aggregations: [{ function: 'count', alias: 'n' }] }), 'previous_checksum'], + ['aggregate: filter', 'filter', (b) => b.aggregate!('sys_metadata', { where: { checksum: HASH_M }, aggregations: [{ function: 'count', alias: 'n' }] }), 'checksum'], + ]; + for (const [label, param, run, field] of CASES) { + it(`${label} → INVALID_FIELD / 400, the engine never asked`, async () => { + const { bridge, engine } = bridgeAs(ADMIN); + let caught: any; + try { + await run(bridge); + } catch (err) { + caught = err; + } + expect(caught, 'the call was refused').toBeInstanceOf(Error); + expect(caught.code).toBe('INVALID_FIELD'); + expect(caught.status).toBe(400); + expect(caught.param).toBe(param); + expect(caught.field).toBe(field); + expect(engine.find).not.toHaveBeenCalled(); + expect(engine.aggregate).not.toHaveBeenCalled(); + }); + } + + it('control: the same shapes on another object\'s checksum column are run', async () => { + const { bridge, engine } = bridgeAs(ADMIN); + await bridge.query('file_blob', { where: { checksum: HASH_M }, orderBy: [{ field: 'checksum', order: 'asc' }] }); + await bridge.aggregate!('file_blob', { groupBy: ['checksum'], aggregations: [{ function: 'count', alias: 'n' }] }); + expect(engine.find).toHaveBeenCalledTimes(1); + expect(engine.aggregate).toHaveBeenCalledTimes(1); + }); +}); + +describe('[#21207] stdio reader: a member stays refused', () => { + it('query and get answer the engine\'s PERMISSION_DENIED', async () => { + const { bridge } = bridgeAs(MEMBER); + await expect(bridge.query('sys_metadata', {})).rejects.toMatchObject({ code: 'PERMISSION_DENIED', status: 403 }); + await expect(bridge.get('sys_metadata_history', 'h1')).rejects.toMatchObject({ code: 'PERMISSION_DENIED', status: 403 }); + }); +}); + +// --------------------------------------------------------------------------- +// The ADR-0101 record resource, over the transport — the plugin's own wiring +// --------------------------------------------------------------------------- + +async function openStdio(server: any) { + const stdin = new PassThrough(); + const stdout = new PassThrough(); + const transport = new StdioServerTransport(stdin, stdout); + await server.connect(transport); + let nextId = 1; + let buffered = ''; + const waiting = new Map void>(); + stdout.on('data', (chunk: Buffer | string) => { + buffered += String(chunk); + let newline = buffered.indexOf('\n'); + while (newline >= 0) { + const line = buffered.slice(0, newline).trim(); + buffered = buffered.slice(newline + 1); + newline = buffered.indexOf('\n'); + if (!line) continue; + const frame = JSON.parse(line); + if (typeof frame.id === 'number') waiting.get(frame.id)?.(frame); + } + }); + const rpc = (method: string, params?: unknown) => + new Promise((resolve, reject) => { + const id = nextId++; + const giveUp = setTimeout(() => reject(new Error(`stdio: no answer to ${method}`)), 5_000); + waiting.set(id, (frame) => { + clearTimeout(giveUp); + resolve(frame); + }); + stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, ...(params ? { params } : {}) })}\n`); + }); + await rpc('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'content-hash-pin', version: '0.0.0' } }); + stdin.write(`${JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' })}\n`); + return { rpc, close: () => transport.close().catch(() => {}) }; +} + +async function readResource(engine: Record, uri: string): Promise> { + const registry = new Map([['metadata', fakeMetadata()], ['objectql', engine]]); + const ctx = { + registerService: vi.fn((name: string, service: unknown) => registry.set(name, service)), + getService: vi.fn((name: string) => { + if (!registry.has(name)) throw new Error(`Service "${name}" not found`); + return registry.get(name); + }), + replaceService: vi.fn(), + getServices: vi.fn(() => registry), + hook: vi.fn(), + trigger: vi.fn(async () => {}), + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + getKernel: vi.fn(() => ({})), + }; + const plugin = new MCPServerPlugin({ autoStart: true }); + await plugin.init(ctx as any); + const runtime = registry.get('mcp') as MCPServerRuntime; + vi.spyOn(runtime, 'start').mockResolvedValue(undefined); + await plugin.start(ctx as any); + const session = await openStdio(runtime.server); + try { + const frame = await session.rpc('resources/read', { uri }); + expect(frame.error, `resources/read ${uri} failed at the protocol level`).toBeUndefined(); + return JSON.parse(frame.result.contents[0].text); + } finally { + await session.close(); + } +} + +describe('[#21207] stdio record resource: the keyed hash, over the transport', () => { + const originalEnv = { ...process.env }; + beforeEach(() => { + process.env = { ...originalEnv }; + delete process.env.OS_MCP_SERVER_TRANSPORT; + delete process.env.OS_MCP_STDIO_ENABLED; + process.env.OS_MCP_STDIO_API_KEY = 'osk_stored_content_hash_pin'; + }); + afterEach(() => { + process.env = { ...originalEnv }; + vi.restoreAllMocks(); + }); + + it('administrator: both tables serve keyed values, no stored hash', async () => { + const m = await readResource(fakeEngine(), 'objectstack://objects/sys_metadata/records/m1'); + expect(m.checksum).toBe(await keyedDigest(HASH_M)); + const h = await readResource(fakeEngine(), 'objectstack://objects/sys_metadata_history/records/h1'); + expect(h.checksum).toMatch(KEYED); + expect(h.previous_checksum).toMatch(KEYED); + noStoredHash([m, h]); + }); + + it('no provider: the resource omits both columns', async () => { + const h = await readResource(fakeEngine({ provider: false }), 'objectstack://objects/sys_metadata_history/records/h1'); + expect('checksum' in h).toBe(false); + expect('previous_checksum' in h).toBe(false); + expect(h.name).toBe('v'); + }); +}); + +describe('[#21207] stdio reader: the history change note that quotes a stored hash', () => { + const QUOTED = `sha256:${'4'.repeat(64)}`; + const NOTE_ROW = { id: 'h_note', type: 'view', name: 'v', metadata: '{"name":"v"}', checksum: QUOTED, previous_checksum: null, change_note: `publish draft (hash ${QUOTED})` }; + + it('is served with the quote keyed, and withheld with no provider', async () => { + TABLES.sys_metadata_history!.push(NOTE_ROW); + try { + const keyed = (await bridgeAs(ADMIN).bridge.get('sys_metadata_history', 'h_note')) as Record; + expect(keyed.change_note).toBe(`publish draft (hash ${await keyedDigest(QUOTED)})`); + const bare = (await bridgeAs(ADMIN, { provider: false }).bridge.get('sys_metadata_history', 'h_note')) as Record; + expect(bare.change_note).toBe('publish draft (hash (withheld))'); + } finally { + TABLES.sys_metadata_history!.pop(); + } + }); + + it('a filter or sort on the change note is refused (INVALID_FIELD / 400)', async () => { + const { bridge, engine } = bridgeAs(ADMIN); + for (const run of [ + () => bridge.query('sys_metadata_history', { where: { change_note: 'x' } }), + () => bridge.query('sys_metadata_history', { orderBy: [{ field: 'change_note', order: 'asc' }] }), + ]) { + await expect(run()).rejects.toMatchObject({ code: 'INVALID_FIELD', status: 400, field: 'change_note' }); + } + expect(engine.find).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/mcp/src/stdio-data-bridge.ts b/packages/mcp/src/stdio-data-bridge.ts index 24dd281d8d5..db03321184f 100644 --- a/packages/mcp/src/stdio-data-bridge.ts +++ b/packages/mcp/src/stdio-data-bridge.ts @@ -128,8 +128,19 @@ export interface StdioDataBridgeDeps { * live session, and a bridge built once at boot would outlive it. */ resolvePrincipal: () => Promise; + /** + * [#21207] The registered crypto provider's keyed digest + * (`ICryptoProvider.keyedDigest`, reached through the engine's + * `getKeyedDigest` accessor), read per call — a host registers the provider + * after the kernel starts. `undefined`, or a resolver answering `undefined`, + * means none: the stored content-hash columns are then not served at all. + */ + keyedDigest?: () => StoredHashDigest | undefined; } +/** The keyed-digest primitive the stored content-hash columns are served through. */ +export type StoredHashDigest = (plain: string) => Promise; + /** An object definition as `IMetadataService.getObject` hands it back. */ interface ObjectDef { name: string; @@ -422,6 +433,140 @@ export function storedMetadataBodyRefusal( return undefined; } +/** + * [#21207] The stored CONTENT-HASH columns of the same two tables: `checksum` + * (both) and the history table's `previous_checksum` (the parent's hash). + * + * Each is the canonical SHA-256 of the WHOLE stored body — withheld credential + * material included — kept as-is at rest. The engine returns it as stored, so + * this engine-only reader served it raw beside the projected body: an offline + * verifier for a guess at the withheld material, and an online one when + * filtered on. Per the maintainer's ruling on #21207 this transport serves the + * crypto provider's KEYED digest of it ({@link serveStoredMetadataHashes}) and + * refuses to evaluate it ({@link storedMetadataHashRefusal}) — the protocol + * data door's answer for the same columns + * (`@objectstack/metadata-protocol`'s `STORED_METADATA_HASH_COLUMNS`, which this + * package cannot import; `stored-metadata-body-family.pin.test.ts` pins the + * list to the object definitions). + */ +export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); + +/** + * [#21207] The history table's change note, which can QUOTE a stored content + * hash (`publish draft (hash …)` on rows written before the publish door stated + * its own message): served with each quote in its served form, never evaluated. + */ +export const STORED_METADATA_HASH_NOTE_COLUMN = 'change_note'; + +/** Every column that holds or can quote a stored content hash. */ +const HASH_BEARING_COLUMNS: readonly string[] = [...STORED_METADATA_HASH_COLUMNS, STORED_METADATA_HASH_NOTE_COLUMN]; + +/** An unkeyed content hash quoted in free text (the keyed form's `hmac-sha256:` prefix is not one). */ +const QUOTED_STORED_HASH = /(? { + const quoted = text.match(QUOTED_STORED_HASH); + if (!quoted) return text; + const served = new Map(); + for (const stored of new Set(quoted)) served.set(stored, digest ? await digest(stored) : '(withheld)'); + return text.replace(QUOTED_STORED_HASH, (stored) => served.get(stored) as string); +} + +/** The hash-bearing column a field reference reaches — the column, or a dotted path headed by it. */ +function hashColumnOf(field: unknown): string | undefined { + if (typeof field !== 'string') return undefined; + const head = field.split('.')[0] as string; + return HASH_BEARING_COLUMNS.includes(head) ? head : undefined; +} + +/** + * [#21207] The refusal for a read of a stored-metadata table whose call would + * EVALUATE a content-hash column — a grouping (whose keys would serve the + * stored values), a filter (a guessed hash matches its row) or a sort — or + * `undefined` when none is named. Same envelope as + * {@link storedMetadataBodyRefusal} (`INVALID_FIELD` / 400, naming the field, + * the object and the part), judged in the same order, and naming the columns + * that remain usable. + */ +export function storedMetadataHashRefusal( + object: string, + opts: { + where?: unknown; + orderBy?: ReadonlyArray<{ field?: unknown }>; + groupBy?: ReadonlyArray; + aggregations?: ReadonlyArray<{ field?: unknown; filter?: unknown }>; + }, +): McpStoredMetadataBodyRefusal | undefined { + if (!isStoredMetadataBodyObject(object)) return undefined; + const make = (param: 'groupBy' | 'filter' | 'sort', column: string): McpStoredMetadataBodyRefusal => { + const doing = param === 'groupBy' ? 'group' : param; + const err = new Error( + `Cannot ${doing} '${object}' by '${column}' (${param}): the query was not run. The '${column}' column ` + + `${column === STORED_METADATA_HASH_NOTE_COLUMN ? 'can quote' : 'holds'} ` + + `the stored content hash of a metadata body, computed over withheld credential material too, so this door ` + + `serves it only in keyed form; ${param === 'filter' + ? 'a filter on it compares a guess against the stored hash row by row, which confirms the guess' + : param === 'groupBy' ? 'a group key would serve the stored value itself' : 'a sort on it orders by the stored values'}. ` + + `Use '${STORED_METADATA_TYPE_COLUMN}', 'name', 'state' or another scalar column instead.`, + ) as McpStoredMetadataBodyRefusal; + err.code = 'INVALID_FIELD'; + err.status = 400; + err.field = column; + err.fields = [column]; + err.object = object; + err.param = param; + return err; + }; + for (const entry of opts.groupBy ?? []) { + const column = hashColumnOf( + entry && typeof entry === 'object' && !Array.isArray(entry) ? (entry as { field?: unknown }).field : entry, + ); + if (column) return make('groupBy', column); + } + const aggregations = Array.isArray(opts.aggregations) ? opts.aggregations : []; + for (const field of [...filterFieldKeys(opts.where), ...aggregations.flatMap((a) => filterFieldKeys(a?.filter))]) { + const column = hashColumnOf(field); + if (column) return make('filter', column); + } + for (const entry of Array.isArray(opts.orderBy) ? opts.orderBy : []) { + const column = hashColumnOf(entry?.field); + if (column) return make('sort', column); + } + return undefined; +} + +/** + * [#21207] Serve one row of `object` with its stored content-hash columns in + * their served form: each the keyed digest of the stored value, a `null` kept + * `null` — and both columns OMITTED when no crypto provider is registered. ⛔ + * Never the stored value. Rows of every other object, and rows carrying + * neither column, are returned by reference. A failing digest is not caught: + * the read fails rather than serving what it exists to replace. + * + * Exported within the package for the ADR-0101 record resource (`plugin.ts`), + * which serves through the same helpers as `bridge.get`. + */ +export async function serveStoredMetadataHashes( + object: string, + row: T, + digest: StoredHashDigest | undefined, +): Promise { + if (!isStoredMetadataBodyObject(object) || !row || typeof row !== 'object' || Array.isArray(row)) return row; + const record = row as Record; + if (!HASH_BEARING_COLUMNS.some((column) => column in record)) return row; + const out: Record = { ...record }; + for (const column of STORED_METADATA_HASH_COLUMNS) { + if (!(column in out)) continue; + const stored = out[column]; + if (!digest || (stored !== null && typeof stored !== 'string')) delete out[column]; + else out[column] = stored === null ? null : await digest(stored); + } + const note = out[STORED_METADATA_HASH_NOTE_COLUMN]; + if (typeof note === 'string') out[STORED_METADATA_HASH_NOTE_COLUMN] = await serveStoredHashTokens(note, digest); + return out as T; +} + /** * The field projection to hand the engine for a read of `object`, given the * caller's own. @@ -486,7 +631,7 @@ export function serveStoredMetadataRows( * honoured rather than re-decided. */ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge { - const { engine, metadataService, resolvePrincipal } = deps; + const { engine, metadataService, resolvePrincipal, keyedDigest } = deps; const bridge: McpDataBridge = { async listObjects(): Promise { @@ -542,6 +687,9 @@ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge // on a stored metadata body is refused rather than evaluated. const bodyRefusal = storedMetadataBodyRefusal(object, { where: opts?.where, orderBy: opts?.orderBy }); if (bodyRefusal) throw bodyRefusal; + // [#21207] …and the same for a stored content-hash column. + const hashRefusal = storedMetadataHashRefusal(object, { where: opts?.where, orderBy: opts?.orderBy }); + if (hashRefusal) throw hashRefusal; const read = storedMetadataBodyReadFields(object, opts?.fields); const query: Record = {}; if (opts?.where) query.where = opts.where; @@ -549,10 +697,12 @@ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge if (opts?.orderBy) query.orderBy = opts.orderBy; if (typeof opts?.limit === 'number') query.limit = opts.limit; if (typeof opts?.offset === 'number') query.offset = opts.offset; - const records = serveStoredMetadataRows( - object, - unwrapRows(await engine.find(object, query, { context })), - { dropType: read.addedType }, + // [#21207] The content-hash columns are served keyed, or not at all. + const digest = keyedDigest?.(); + const records = await Promise.all( + serveStoredMetadataRows(object, unwrapRows(await engine.find(object, query, { context })), { + dropType: read.addedType, + }).map((row) => serveStoredMetadataHashes(object, row, digest)), ); return { object, records, total: records.length }; }, @@ -562,8 +712,13 @@ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge await enforceApiExposure(metadataService, object, GATED_ACTIONS.get, context); // `null` rather than a throw: `get_record` owns the not-found wording on // this path and already branches on a nullish record. - // [#21207] A stored metadata body is served as its type's projection. - return serveStoredMetadataRow(object, await findById(engine, object, id, context)); + // [#21207] A stored metadata body is served as its type's projection, + // and its stored content hash keyed (or not at all). + return serveStoredMetadataHashes( + object, + serveStoredMetadataRow(object, await findById(engine, object, id, context)), + keyedDigest?.(), + ); }, async create(object, data) { @@ -643,6 +798,13 @@ export function createStdioDataBridge(deps: StdioDataBridgeDeps): McpDataBridge aggregations: opts?.aggregations, }); if (bodyRefusal) throw bodyRefusal; + // [#21207] …and a grouping or filter on a stored content-hash column. + const hashRefusal = storedMetadataHashRefusal(object, { + where: opts?.where, + groupBy: opts?.groupBy, + aggregations: opts?.aggregations, + }); + if (hashRefusal) throw hashRefusal; // No casts: `McpDataBridge.aggregate` declares the engine's own // `EngineAggregateOptions` slices since #8032, so the honest call // compiles — the two `as unknown as` casts this line used to carry diff --git a/packages/metadata-protocol/src/metadata-redaction.ts b/packages/metadata-protocol/src/metadata-redaction.ts index 9d7d8783ec7..b4250d48d73 100644 --- a/packages/metadata-protocol/src/metadata-redaction.ts +++ b/packages/metadata-protocol/src/metadata-redaction.ts @@ -66,6 +66,7 @@ * belongs on that list first; that file is `packages/spec`'s to change. */ +import { createHmac, randomBytes } from 'node:crypto'; import { getMetadataTypeRedactor } from '@objectstack/spec/kernel'; import type { MetadataTypeRedactor } from '@objectstack/spec/kernel'; // [#21120] The family-wide stored-metadata-body primitives — the object set, @@ -774,3 +775,279 @@ export function storedMetadataBodyPredicateRefusal( if (namesBody(opts.sortFields)) return make('sort', 'sort'); return undefined; } + +// --------------------------------------------------------------------------- +// The stored CONTENT HASH of the same rows: served keyed, never evaluated (#21207) +// --------------------------------------------------------------------------- + +/** + * [#21207] The columns of a {@link isStoredMetadataBodyObject} table that hold + * a stored CONTENT HASH of the body: `checksum` (both tables) and the history + * table's `previous_checksum` (the parent's hash). + * + * The hash is the canonical SHA-256 of the WHOLE stored body — withheld + * credential material included — and it stays that at rest: the repository's + * canonical-hashing invariant, the optimistic lock and the parent links are + * untouched (maintainer ruling B on #21207). What changes is what a caller is + * given. Served raw beside the projected body, it is an OFFLINE VERIFIER: a + * guess at the withheld material, hashed together with the served rest of the + * body, reproduces it exactly when the guess is right. Evaluated as a + * predicate, it is an online one. So every door that serves it serves the + * crypto provider's KEYED digest of the stored value ({@link servedContentHash}), + * every door that takes a version token back compares it in that same form, and + * a filter, sort, grouping or search on it is refused + * ({@link storedMetadataHashEvaluateRefusal}, {@link storedMetadataSearchRefusal}). + * + * Kept beside the body column's family primitives' consumer rather than in + * `@objectstack/spec/kernel` with them: that is the family's natural home, but + * outside the surface this card claimed. The surfaces that cannot import this + * package (`@objectstack/mcp`, `@objectstack/plugin-audit`, + * `@objectstack/service-analytics`) each name the same two columns, and + * `stored-metadata-body-family.pin.test.ts` pins this list to the columns the + * two object definitions declare. + */ +export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); + +/** + * [#21207] The history table's free-text change note — which can QUOTE a stored + * content hash: with no message of its own, a draft's promotion recorded + * `publish draft (hash )`. The protocol now always + * states a hash-free message, so no new row carries one; rows already written + * do, so the column is served with each quoted hash in its served form + * ({@link serveStoredHashTokens}) and is never evaluated, exactly as the hash + * columns are. + */ +export const STORED_METADATA_HASH_NOTE_COLUMN = 'change_note'; + +/** Every column of a stored-metadata table that holds or can quote a stored content hash. */ +export const STORED_METADATA_HASH_BEARING_COLUMNS: readonly string[] = Object.freeze([ + ...STORED_METADATA_HASH_COLUMNS, + STORED_METADATA_HASH_NOTE_COLUMN, +]); + +/** An unkeyed content hash quoted in free text (the keyed form's `hmac-sha256:` prefix is not one). */ +const QUOTED_STORED_HASH = /(? { + const quoted = text.match(QUOTED_STORED_HASH); + if (!quoted) return text; + const served = new Map(); + for (const stored of new Set(quoted)) served.set(stored, digest ? await digest(stored) : '(withheld)'); + return text.replace(QUOTED_STORED_HASH, (stored) => served.get(stored) as string); +} + +/** The keyed-digest primitive of the registered crypto provider (`ICryptoProvider.keyedDigest`). */ +export type StoredHashDigest = (plain: string) => Promise; + +/** + * [#21207] The process key {@link ephemeralStoredHashDigest} keys under: 32 + * random bytes, drawn on first use, held only in this module, never written, + * logged or served. + */ +let ephemeralDigestKey: Buffer | undefined; + +/** + * [#21207] The keyed digest the `/meta` doors serve and compare metadata + * version tokens under while NO crypto provider is registered: `hmac-sha256:` + * plus hex, the provider contract's own output shape, under a process-scoped + * ephemeral key. + * + * Why a key and not "serve nothing": a version token is an optimistic lock. + * Serving none hands every caller the same empty token, and a client that + * (rightly) sends no pin for an empty token turns every pinned write into an + * unpinned one, so the lock fails OPEN without a word. A token keyed under a + * secret nobody outside this process holds keeps the three properties the lock + * needs: it differs when the content differs; it is never the unkeyed stored + * hash, so it confirms no guess at withheld material offline; and no empty or + * withheld value ever equals it. + * + * What it costs: the key dies with the process. A token held across a restart, + * or across the moment a host registers a real provider (the doors read the + * provider per use), names no current version and is refused once with + * `409 METADATA_CONFLICT`; the next read or receipt serves the current one. + * Every protocol in one process shares this key, so per-environment protocols + * answer one another's tokens. + */ +export const ephemeralStoredHashDigest: StoredHashDigest = async (plain: string): Promise => { + ephemeralDigestKey ??= randomBytes(32); + return `hmac-sha256:${createHmac('sha256', ephemeralDigestKey).update(plain, 'utf8').digest('hex')}`; +}; + +/** + * The form a stored content hash is SERVED in: the keyed digest of the stored + * value under a server-held key; `null` when nothing is stored (a delete + * event, a first version's parent); `undefined` (WITHHELD) when the caller + * holds no digest, or when the stored value is not a string this function can + * judge. ⛔ Never the stored value itself. + * + * A failing digest is not caught: a provider that cannot compute it fails the + * read rather than serving what it exists to replace. + */ +export async function servedContentHash( + stored: unknown, + digest: StoredHashDigest | undefined, +): Promise { + if (stored === null || stored === undefined) return null; + if (typeof stored !== 'string' || !digest) return undefined; + return digest(stored); +} + +/** + * Serve one row of a stored-metadata table with its content-hash columns in + * their served form ({@link servedContentHash}): keyed, `null` kept `null`, and + * the column OMITTED when the value is withheld — and, when the caller holds no + * digest, both columns omitted outright. A row of any other object, and a row + * carrying neither column, is returned by reference. + */ +export async function serveStoredMetadataHashColumns( + object: string, + row: T, + digest: StoredHashDigest | undefined, +): Promise { + if (!isStoredMetadataBodyObject(object) || !isPlainRecord(row)) return row; + if (!STORED_METADATA_HASH_BEARING_COLUMNS.some((column) => column in row)) return row; + const out: Record = { ...row }; + for (const column of STORED_METADATA_HASH_COLUMNS) { + if (!(column in out)) continue; + // No digest: the column is not served at all — a `null` included, so + // a reader cannot tell a withheld hash from an absent one either. + const served = digest ? await servedContentHash(out[column], digest) : undefined; + if (served === undefined) delete out[column]; + else out[column] = served; + } + const note = out[STORED_METADATA_HASH_NOTE_COLUMN]; + if (typeof note === 'string') out[STORED_METADATA_HASH_NOTE_COLUMN] = await serveStoredHashTokens(note, digest); + return out as T; +} + +/** {@link serveStoredMetadataHashColumns} over the rows of one read. Non-array input passes through. */ +export async function serveStoredMetadataHashColumnRows( + object: string, + rows: T[], + digest: StoredHashDigest | undefined, +): Promise { + if (!Array.isArray(rows) || !isStoredMetadataBodyObject(object)) return rows; + return Promise.all(rows.map((row) => serveStoredMetadataHashColumns(object, row, digest))); +} + +/** The content-hash column a field reference reaches — the column, or a dotted path headed by it. */ +function hashColumnOf(field: unknown): string | undefined { + if (typeof field !== 'string') return undefined; + const head = field.split('.')[0] as string; + return STORED_METADATA_HASH_BEARING_COLUMNS.includes(head) ? head : undefined; +} + +/** The columns a refusal on these tables points the caller at instead. */ +const USABLE_COLUMNS = `'${STORED_TYPE_COLUMN}', 'name', 'state' or another scalar column`; + +/** + * [#21207] The data door's refusal to EVALUATE a content-hash column of a + * stored-metadata table — or the history table's change note, which can quote + * one ({@link STORED_METADATA_HASH_BEARING_COLUMNS}) — a grouping (whose keys would serve the stored + * values), a filter (an online verifier: a guessed hash matches exactly one + * row) or a sort (an order over the same values) — or `undefined` when none is + * named. Maintainer ruling A on #21207's second execution fork. + * + * The family's existing body-column refusals' shape, extended to these columns + * rather than a second dialect: `INVALID_FIELD` / 400 naming the field, the + * object and the offending `param`, judged in the data door's order — grouping, + * then filter, then sort — and naming the columns that remain usable. A dotted + * path headed by a hash column is caught too. + */ +export function storedMetadataHashEvaluateRefusal( + object: string, + opts: { groupBy?: unknown; filterFields?: readonly unknown[]; sortFields?: readonly unknown[] }, +): Error | undefined { + if (!isStoredMetadataBodyObject(object)) return undefined; + const make = (param: 'groupBy' | 'filter' | 'sort', column: string, position?: string): Error => { + const doing = param === 'groupBy' ? 'group' : param; + const why = param === 'filter' + ? 'a filter on it compares a guess against the stored hash row by row, which confirms the guess' + : param === 'groupBy' + ? 'a group key would serve the stored value itself' + : 'a sort on it orders by the stored values'; + const err: any = new Error( + `Cannot ${doing} '${object}' by '${column}' (${position ?? param}): the query was not run. The ` + + `'${column}' column ${column === STORED_METADATA_HASH_NOTE_COLUMN + ? 'can quote the stored content hash of a metadata body' + : 'holds the stored content hash of a metadata body'}, computed over withheld ` + + `credential material too, so this door serves it only in keyed form; ${why}. ` + + `${doing === 'group' ? 'Group' : doing === 'filter' ? 'Filter' : 'Sort'} by ${USABLE_COLUMNS} instead.`, + ); + err.code = 'INVALID_FIELD'; + err.status = 400; + err.field = column; + err.fields = [column]; + err.object = object; + err.param = param; + return err; + }; + if (Array.isArray(opts.groupBy)) { + for (let i = 0; i < opts.groupBy.length; i += 1) { + const entry = opts.groupBy[i]; + const objectForm = isPlainRecord(entry); + const column = hashColumnOf(objectForm ? entry.field : entry); + if (column) return make('groupBy', column, objectForm ? `groupBy[${i}].field` : `groupBy[${i}]`); + } + } + for (const field of opts.filterFields ?? []) { + const column = hashColumnOf(field); + if (column) return make('filter', column); + } + for (const field of opts.sortFields ?? []) { + const column = hashColumnOf(field); + if (column) return make('sort', column); + } + return undefined; +} + +/** + * [#21207] The columns of a stored-metadata table a `search` never scans: the + * body column, the content-hash columns and the change note that can quote one. A search is a substring filter + * evaluated server-side over every scanned column, so over these columns it is + * the same verifier a filter is — over the stored hash, and over the stored + * body (a withheld credential rebuilt by prefix probing) — and the engine's + * auto-default search set includes every one of them, since all three are + * text columns. + */ +export const STORED_METADATA_UNSEARCHABLE_COLUMNS: readonly string[] = Object.freeze([ + STORED_BODY_COLUMN, + ...STORED_METADATA_HASH_BEARING_COLUMNS, +]); + +/** + * The refusal for an EXPLICIT search field list (`searchFields`, or the + * object-form `search.fields`) on a stored-metadata table that names a column + * of {@link STORED_METADATA_UNSEARCHABLE_COLUMNS}, or `undefined`. Same + * envelope as the evaluate refusals: `INVALID_FIELD` / 400. + */ +export function storedMetadataSearchRefusal( + object: string, + requested: readonly string[], + param: string, +): Error | undefined { + if (!isStoredMetadataBodyObject(object)) return undefined; + const column = requested.find((name) => STORED_METADATA_UNSEARCHABLE_COLUMNS.includes(name)); + if (column === undefined) return undefined; + const err: any = new Error( + `Cannot search '${object}' in '${column}' (${param}): the query was not run. A search evaluates ` + + `every column it scans row by row, and the '${column}' column holds ${column === STORED_BODY_COLUMN + ? 'a stored metadata body with credential material this door withholds' + : column === STORED_METADATA_HASH_NOTE_COLUMN + ? 'a change note that can quote a stored content hash, which this door serves only in keyed form' + : 'the stored content hash of a metadata body, which this door serves only in keyed form'}, so a ` + + `search over it would answer guesses about withheld values. Search ${USABLE_COLUMNS} instead.`, + ); + err.code = 'INVALID_FIELD'; + err.status = 400; + err.field = column; + err.fields = [column]; + err.object = object; + err.param = param; + return err; +} diff --git a/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts b/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts new file mode 100644 index 00000000000..1559095ee71 --- /dev/null +++ b/packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts @@ -0,0 +1,309 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21207] Exit two at the generic data door — the two stored content-hash + * columns of `sys_metadata` / `sys_metadata_history` (`checksum`, and the + * history table's `previous_checksum`). + * + * A stored content hash is computed over the WHOLE stored body, withheld + * credential material included. Served raw beside the projected body, it is an + * offline verifier: a guess at the withheld material, hashed with the rest of + * the served body, either reproduces it or not. Evaluated as a predicate it is + * an online one. So, per the maintainer's ruling on this card: + * + * - SERVED: each column is the crypto provider's keyed digest of the stored + * value — neither the stored value nor a recomputation, stable across reads, + * moving on any body change including a credential-only one; a `null` stays + * `null`; with no provider the key is the process-scoped ephemeral one; + * - EVALUATED: a filter, sort or grouping on either column is refused before + * the engine is asked — `INVALID_FIELD` / 400, the shape the body column's + * refusals already answer — and a search never scans them (nor the body + * column): an explicit `searchFields` naming one is refused, an implicit one + * is narrowed to the other columns. + * + * Every other object — including one with its own `checksum` column — is served + * and evaluated exactly as before. + */ + +import { createHmac } from 'node:crypto'; +import { describe, expect, it, vi } from 'vitest'; +import { assertEngineFindOnePredicate, hashSpec, SysMetadataHistoryObject, SysMetadataObject } from '@objectstack/metadata-core'; +import { ObjectStackProtocolImplementation } from './protocol.js'; + +const keyedDigest = async (plain: string): Promise => + `hmac-sha256:${createHmac('sha256', 'data-door-test-key').update(plain, 'utf8').digest('hex')}`; +const KEYED = /^hmac-sha256:[0-9a-f]{64}$/; + +const CRED_A = 'data-door-cred-a'; +const CRED_B = 'data-door-cred-b'; +const dsBody = (cred: string) => ({ + name: 'edge_cache', + label: 'Edge Cache', + driver: 'turso', + config: { url: 'file:/var/data/edge.db', encryptionKey: cred }, +}); +const viewBody = { name: 'all_tasks', label: 'All Tasks', type: 'grid' }; + +const SYS_METADATA_ROWS = [ + { id: 'm_a', type: 'datasource', name: 'edge_cache', state: 'active', metadata: JSON.stringify(dsBody(CRED_A)), checksum: hashSpec(dsBody(CRED_A)) }, + { id: 'm_b', type: 'datasource', name: 'edge_cache_b', state: 'active', metadata: JSON.stringify(dsBody(CRED_B)), checksum: hashSpec(dsBody(CRED_B)) }, + { id: 'm_view', type: 'view', name: 'all_tasks', state: 'active', metadata: JSON.stringify(viewBody), checksum: hashSpec(viewBody) }, +]; +const HISTORY_ROWS = [ + { id: 'h_1', type: 'view', name: 'all_tasks', version: 1, operation_type: 'create', metadata: JSON.stringify(viewBody), checksum: hashSpec(viewBody), previous_checksum: null }, + { id: 'h_2', type: 'view', name: 'all_tasks', version: 2, operation_type: 'update', metadata: JSON.stringify(viewBody), checksum: hashSpec({ ...viewBody, label: 'x' }), previous_checksum: hashSpec(viewBody) }, +]; +/** An ordinary object with a `checksum` column of its own — not a stored metadata body. */ +const FILE_SCHEMA = { + name: 'file_blob', + fields: { name: { name: 'name', type: 'text' }, checksum: { name: 'checksum', type: 'text' } }, +}; +const FILE_ROWS = [{ id: 'f_1', name: 'blob', checksum: 'sha256:0000' }]; + +const ROWS: Record[]> = { + sys_metadata: SYS_METADATA_ROWS, + sys_metadata_history: HISTORY_ROWS, + file_blob: FILE_ROWS, +}; +const SCHEMAS: Record = { + sys_metadata: SysMetadataObject, + sys_metadata_history: SysMetadataHistoryObject, + file_blob: FILE_SCHEMA, +}; + +function matches(row: Record, where: unknown): boolean { + if (!where || typeof where !== 'object') return true; + return Object.entries(where as Record).every(([k, v]) => { + if (k.startsWith('$')) throw new Error(`stub engine: combinator '${k}' is not implemented`); + if (v !== null && typeof v === 'object') throw new Error(`stub engine: operator value on '${k}' is not implemented`); + return row[k] === v; + }); +} + +function makeProtocol(opts: { provider?: boolean } = {}) { + const find = vi.fn(async (object: string, o: any) => (ROWS[object] ?? []).filter((r) => matches(r, o?.where)).map((r) => ({ ...r }))); + const findOne = vi.fn(async (object: string, o: any) => { + assertEngineFindOnePredicate(object, o); + const hit = (ROWS[object] ?? []).find((r) => matches(r, o?.where)); + return hit ? { ...hit } : null; + }); + const aggregate = vi.fn(async () => [{ type: 'datasource', n: 2 }]); + const count = vi.fn(async () => 0); + const engine: any = { registry: { getObject: (n: string) => SCHEMAS[n] }, find, findOne, count, aggregate }; + if (opts.provider !== false) engine.getKeyedDigest = () => keyedDigest; + return { p: new ObjectStackProtocolImplementation(engine), find, findOne, aggregate, count }; +} + +const byId = (records: any[], id: string) => records.find((r) => r.id === id); + +async function refusal(run: () => Promise): Promise { + let caught: any; + let resolved = false; + try { + await run(); + resolved = true; + } catch (e) { + caught = e; + } + expect(resolved, 'expected a refusal, but the query ran').toBe(false); + return caught; +} + +describe('[#21207] data door — the content-hash columns are served keyed', () => { + it('list read: keyed, not stored, not a recomputation from the served body; stable across reads', async () => { + const { p } = makeProtocol(); + const first: any = await p.findData({ object: 'sys_metadata', query: {} }); + const second: any = await p.findData({ object: 'sys_metadata', query: {} }); + + for (const row of SYS_METADATA_ROWS) { + const served = byId(first.records, row.id); + expect(served.checksum).toMatch(KEYED); + expect(served.checksum).not.toBe(row.checksum); + const servedBody = JSON.parse(served.metadata); + expect(served.checksum).not.toBe(hashSpec(servedBody)); + expect(served.checksum).toBe(await keyedDigest(row.checksum)); + expect(byId(second.records, row.id).checksum).toBe(served.checksum); + } + }); + + it('a credential-only difference: the served bodies are identical, the served hashes are not', async () => { + const { p } = makeProtocol(); + const res: any = await p.findData({ object: 'sys_metadata', query: { type: 'datasource' } }); + const a = byId(res.records, 'm_a'); + const b = byId(res.records, 'm_b'); + const { name: _na, ...bodyA } = JSON.parse(a.metadata); + const { name: _nb, ...bodyB } = JSON.parse(b.metadata); + expect(bodyA).toEqual(bodyB); + expect(a.checksum).not.toBe(b.checksum); + expect(JSON.stringify(res.records)).not.toContain(CRED_A); + }); + + it('by-id read: keyed, equal to the list read', async () => { + const { p } = makeProtocol(); + const got: any = await p.getData({ object: 'sys_metadata', id: 'm_a' }); + const listed: any = await p.findData({ object: 'sys_metadata', query: {} }); + expect(got.record.checksum).toMatch(KEYED); + expect(got.record.checksum).toBe(byId(listed.records, 'm_a').checksum); + }); + + it('history table: both columns keyed, a null parent stays null', async () => { + const { p } = makeProtocol(); + const res: any = await p.findData({ object: 'sys_metadata_history', query: {} }); + expect(byId(res.records, 'h_1').previous_checksum).toBeNull(); + expect(byId(res.records, 'h_2').previous_checksum).toBe(await keyedDigest(HISTORY_ROWS[1]!.previous_checksum as string)); + for (const row of res.records) expect(row.checksum).toMatch(KEYED); + const got: any = await p.getData({ object: 'sys_metadata_history', id: 'h_2' }); + expect(got.record.previous_checksum).toMatch(KEYED); + }); + + it('no provider: both columns keyed under the process key, never stored, stable across reads', async () => { + const { p } = makeProtocol({ provider: false }); + const list: any = await p.findData({ object: 'sys_metadata', query: {} }); + const again: any = await p.findData({ object: 'sys_metadata', query: {} }); + for (const row of SYS_METADATA_ROWS) { + const served = byId(list.records, row.id); + expect(served.checksum).toMatch(KEYED); + expect(served.checksum).not.toBe(row.checksum); + expect(served.checksum).not.toBe(await keyedDigest(row.checksum)); + expect(byId(again.records, row.id).checksum).toBe(served.checksum); + expect(typeof served.name).toBe('string'); + } + const hist: any = await p.findData({ object: 'sys_metadata_history', query: {} }); + expect(byId(hist.records, 'h_1').previous_checksum).toBeNull(); + for (const row of hist.records) expect(row.checksum).toMatch(KEYED); + const got: any = await p.getData({ object: 'sys_metadata', id: 'm_view' }); + expect(got.record.checksum).toBe(byId(list.records, 'm_view').checksum); + expect(got.record.name).toBe('all_tasks'); + }); + + it('control: an object outside the family keeps its own checksum column, raw', async () => { + const { p } = makeProtocol(); + const res: any = await p.findData({ object: 'file_blob', query: {} }); + expect(res.records[0].checksum).toBe('sha256:0000'); + }); +}); + +describe('[#21207] data door — the content-hash columns are never evaluated', () => { + const cases: Array<[string, string, Record, string]> = [ + ['filter', 'sys_metadata', { filter: JSON.stringify({ checksum: SYS_METADATA_ROWS[0]!.checksum }) }, 'checksum'], + ['implicit filter', 'sys_metadata', { checksum: SYS_METADATA_ROWS[0]!.checksum }, 'checksum'], + ['parent-hash filter', 'sys_metadata_history', { filter: JSON.stringify({ previous_checksum: 'x' }) }, 'previous_checksum'], + ['sort', 'sys_metadata', { sort: 'checksum' }, 'checksum'], + ['descending sort', 'sys_metadata_history', { sort: '-previous_checksum' }, 'previous_checksum'], + ['group', 'sys_metadata', { groupBy: ['checksum'], aggregations: [{ function: 'count', alias: 'n' }] }, 'checksum'], + ['group (object form)', 'sys_metadata_history', { groupBy: [{ field: 'previous_checksum' }], aggregations: [{ function: 'count', alias: 'n' }] }, 'previous_checksum'], + ['aggregation filter', 'sys_metadata', { aggregations: [{ function: 'count', alias: 'n', filter: { checksum: 'x' } }] }, 'checksum'], + ]; + for (const [label, object, query, field] of cases) { + it(`${label} on '${field}' is refused (INVALID_FIELD / 400) and the engine is never asked`, async () => { + const { p, find, aggregate, count } = makeProtocol(); + const err = await refusal(() => p.findData({ object, query })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe(field); + expect(err.object).toBe(object); + expect(find).not.toHaveBeenCalled(); + expect(aggregate).not.toHaveBeenCalled(); + expect(count).not.toHaveBeenCalled(); + }); + } + + it('the refusal names the usable columns', async () => { + const { p } = makeProtocol(); + const err = await refusal(() => p.findData({ object: 'sys_metadata', query: { sort: 'checksum' } })); + expect(err.message).toContain("'type'"); + expect(err.message).toContain("'name'"); + }); + + it('control: filter, sort and group by a scalar column still run', async () => { + const { p, find, aggregate } = makeProtocol(); + await p.findData({ object: 'sys_metadata', query: { type: 'view', sort: 'name' } }); + expect(find).toHaveBeenCalledTimes(1); + await p.findData({ object: 'sys_metadata', query: { groupBy: ['type'], aggregations: [{ function: 'count', alias: 'n' }] } }); + expect(aggregate).toHaveBeenCalledTimes(1); + }); + + it('control: an object outside the family is filterable by its own checksum column', async () => { + const { p, find } = makeProtocol(); + const res: any = await p.findData({ object: 'file_blob', query: { checksum: 'sha256:0000' } }); + expect(find).toHaveBeenCalledTimes(1); + expect(res.records).toHaveLength(1); + }); +}); + +describe('[#21207] data door — a search never scans the hash or body columns of a stored-metadata table', () => { + for (const field of ['checksum', 'metadata']) { + it(`an explicit searchFields naming '${field}' is refused (INVALID_FIELD / 400)`, async () => { + const { p, find } = makeProtocol(); + const err = await refusal(() => p.findData({ object: 'sys_metadata', query: { search: 'abc', searchFields: field } })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe(field); + expect(find).not.toHaveBeenCalled(); + }); + } + + it('an implicit search is narrowed: the engine is handed a field set without them', async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'sys_metadata_history', query: { search: 'abc' } }); + const handed = find.mock.calls[0]![1].searchFields as string[]; + expect(Array.isArray(handed)).toBe(true); + expect(handed.length).toBeGreaterThan(0); + for (const refused of ['checksum', 'previous_checksum', 'metadata']) expect(handed).not.toContain(refused); + expect(handed).toContain('name'); + }); + + it('control: a search outside the family is handed on unchanged', async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'file_blob', query: { search: 'abc' } }); + expect(find.mock.calls[0]![1].searchFields).toBeUndefined(); + }); +}); + +describe('[#21207] data door — the history change note that quotes a stored hash', () => { + const QUOTED = hashSpec({ quoted: true }); + const NOTE_ROWS = [{ id: 'h_note', type: 'view', name: 'all_tasks', version: 3, operation_type: 'publish', metadata: '{}', checksum: QUOTED, previous_checksum: null, change_note: `publish draft (hash ${QUOTED})` }]; + + it('is served with the quote keyed, under the provider\'s key or the process key', async () => { + const saved = [...HISTORY_ROWS]; + HISTORY_ROWS.push(...(NOTE_ROWS as any)); + try { + const { p } = makeProtocol(); + const got: any = await p.getData({ object: 'sys_metadata_history', id: 'h_note' }); + expect(got.record.change_note).toBe(`publish draft (hash ${await keyedDigest(QUOTED)})`); + const bare = makeProtocol({ provider: false }); + const keyed: any = await bare.p.findData({ object: 'sys_metadata_history', query: {} }); + const note = byId(keyed.records, 'h_note'); + // The quote is served as the row's own served hash column. + expect(note.change_note).toBe(`publish draft (hash ${note.checksum})`); + expect(note.checksum).toMatch(KEYED); + expect(JSON.stringify(keyed.records)).not.toContain(QUOTED); + } finally { + HISTORY_ROWS.length = 0; + HISTORY_ROWS.push(...saved); + } + }); + + for (const [label, query] of [ + ['filter', { filter: JSON.stringify({ change_note: 'x' }) }], + ['sort', { sort: 'change_note' }], + ['group', { groupBy: ['change_note'], aggregations: [{ function: 'count', alias: 'n' }] }], + ['explicit search fields', { search: 'abc', searchFields: 'change_note' }], + ] as const) { + it(`${label} on the change note is refused (INVALID_FIELD / 400)`, async () => { + const { p, find, aggregate } = makeProtocol(); + const err = await refusal(() => p.findData({ object: 'sys_metadata_history', query: query as any })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe('change_note'); + expect(find).not.toHaveBeenCalled(); + expect(aggregate).not.toHaveBeenCalled(); + }); + } + + it('an implicit search does not scan the change note', async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'sys_metadata_history', query: { search: 'abc' } }); + expect(find.mock.calls[0]![1].searchFields).not.toContain('change_note'); + }); +}); diff --git a/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts b/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts index fdcdcbf5510..760ac65ae93 100644 --- a/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts +++ b/packages/metadata-protocol/src/protocol.lifecycle-audit-rows.test.ts @@ -458,7 +458,10 @@ describe('[#7748] the audit trail records the whole lifecycle, not only `save`', code: 'metadata_conflict', actor: 'admin', }); - expect(String(denial.note)).toContain('sha256:stale'); + // [#21207] The note names which side was present, never a version + // token or a stored hash: a copy carries no hash (fork three, ruling A). + expect(String(denial.note)).not.toContain('sha256:stale'); + expect(String(denial.note)).toContain('(withheld)'); }); // ── the read door ─────────────────────────────────────────────────────── diff --git a/packages/metadata-protocol/src/protocol.package-publish-audit-rows.test.ts b/packages/metadata-protocol/src/protocol.package-publish-audit-rows.test.ts index e4db9ffeff4..29ccb75e6ac 100644 --- a/packages/metadata-protocol/src/protocol.package-publish-audit-rows.test.ts +++ b/packages/metadata-protocol/src/protocol.package-publish-audit-rows.test.ts @@ -980,7 +980,10 @@ describe('[#8594] a refused publish leaves the INNER verdict, in its own vocabul source: 'protocol.publishPackageDrafts', }); // The note names the losing race, which is the fact an author needs. - expect(String(conflict[0].note)).toContain('sha256:advanced_by_a_rival'); + // [#21207] It names it WITHOUT the stored hashes — a copy never + // carries one (fork three, ruling A): each present side is withheld. + expect(String(conflict[0].note)).toBe('expected parent (withheld) but current is (withheld)'); + expect(String(conflict[0].note)).not.toContain('sha256:advanced_by_a_rival'); expect(publishRows(h, 'denied').map((a) => a.code).sort()) .toEqual(['batch_aborted', 'metadata_conflict']); // The batch reports the refusal in its own envelope too. diff --git a/packages/metadata-protocol/src/protocol.served-content-hash.test.ts b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts new file mode 100644 index 00000000000..2a69d3c06cb --- /dev/null +++ b/packages/metadata-protocol/src/protocol.served-content-hash.test.ts @@ -0,0 +1,464 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21207] Exit two — the `/meta` doors serve a stored content hash only in + * KEYED form, and compare an inbound version token in that same form. + * + * The stored content hash of a metadata body stays the canonical, unkeyed hash + * at rest (the repository's invariant is untouched). Every door that SERVES it + * — the save, publish and rollback receipts, the history read, and the 409 + * conflict refusal — serves the crypto provider's keyed digest of it instead, + * so a caller holding the projected body cannot recompute it and confirm a guess + * about withheld credential material offline. Every door that takes a version + * token back (the save door's and the reset door's optimistic lock) compares it + * in keyed form against the current stored value and hands the STORED value to + * the repository. With no provider registered the key is a process-scoped + * ephemeral one: tokens are still served, so the optimistic lock never fails + * open on an empty token. + * + * Pinned per door, as an administrator would read it: + * - the served value is neither the stored hash nor a recomputation from the + * served body; two reads agree; a change to the body moves it; + * - the keyed token is accepted inbound, the raw stored hash is refused with + * the ADR-0112 envelope (`METADATA_CONFLICT` / 409); + * - the refusal's text and attributes carry the keyed value or none, and the + * decision-audit note it writes carries no hash at all; + * - no provider: tokens keyed under the process key, never empty; an empty, + * withheld, raw or stale token is refused on every door. + * + * The engine is an in-memory double of the stored tables with the repository's + * own read and write shapes (the `protocol.lifecycle-audit-rows.test.ts` double, + * plus the engine's `getKeyedDigest` accessor), so the stored values compared + * against are the ones the real repository writes. + */ + +import { createHmac } from 'node:crypto'; +import { describe, expect, it } from 'vitest'; +import { + assertEngineDeleteDispatch, + assertEngineFindOnePredicate, + assertEngineUpdateDispatch, + hashSpec, +} from '@objectstack/metadata-core'; +import { ObjectStackProtocolImplementation } from './protocol.js'; + +const TEST_KEY = 'served-content-hash-test-key'; +/** A keyed digest with the provider contract's output shape, under a key this file holds. */ +const keyedDigest = async (plain: string): Promise => + `hmac-sha256:${createHmac('sha256', TEST_KEY).update(plain, 'utf8').digest('hex')}`; + +/** An unkeyed content hash — the keyed form's `hmac-sha256:` prefix is not one. */ +const SHA256 = /(?) { + return `${w.type}|${w.name}|${w.organization_id ?? '__env__'}|${w.state ?? 'active'}|${w.package_id ?? '__nopkg__'}`; +} + +function matchesWhere(r: Record, where: Record = {}): boolean { + for (const [k, v] of Object.entries(where)) { + if (k === '$or') { + if (!(v as Array>).some((c) => matchesWhere(r, c))) return false; + continue; + } + // Any other combinator is REFUSED, never read as a field name. + if (k.startsWith('$')) throw new Error(`stub engine: combinator '${k}' is not implemented`); + if (v === undefined) continue; + if (r[k] !== v) return false; + } + return true; +} + +function makeEngine(opts: { provider?: boolean } = {}) { + const rows = new Map(); + const historyRows: Array> = []; + const auditRows: Array> = []; + let nextId = 0; + const findRow = (w: Record): { key: string; row: Row } | null => { + if (w.id !== undefined) { + for (const [k, r] of rows) if (r.id === w.id) return { key: k, row: r }; + return null; + } + if (w.package_id !== undefined) { + const k = keyOf(w); + const r = rows.get(k); + return r ? { key: k, row: r } : null; + } + for (const [k, r] of rows) if (matchesWhere(r, w)) return { key: k, row: r }; + return null; + }; + const engine: any = { + async findOne(table: string, o: { where: Record }) { + assertEngineFindOnePredicate(table, o); + if (table === 'sys_metadata_history') return historyRows.find((h) => matchesWhere(h, o.where)) ?? null; + return findRow(o.where)?.row ?? null; + }, + async find(table: string, o: { where?: Record; limit?: number } = {}) { + const matched = table === 'sys_metadata_audit' + ? auditRows.filter((a) => matchesWhere(a, o.where)) + : table === 'sys_metadata_history' + ? historyRows.filter((h) => matchesWhere(h, o.where)) + : Array.from(rows.values()).filter((r) => matchesWhere(r, o.where)); + // The caller's bound, applied after the filter, by presence. + return typeof o?.limit === 'number' ? matched.slice(0, o.limit) : matched; + }, + async insert(table: string, data: Record) { + nextId += 1; + if (table === 'sys_metadata_audit') { + auditRows.push({ id: `a_${nextId}`, ...data }); + return { id: `a_${nextId}` }; + } + if (table === 'sys_metadata_history') { + historyRows.push({ id: `h_${nextId}`, ...data }); + return { id: `h_${nextId}` }; + } + const row = { id: `r_${nextId}`, ...data } as Row; + rows.set(keyOf(data), row); + return { id: row.id }; + }, + async update(_t: string, data: Record, o: { where: Record }) { + assertEngineUpdateDispatch(data, o); + const found = findRow(o.where); + if (!found) return { id: null }; + const merged = { ...found.row, ...data } as Row; + rows.delete(found.key); + rows.set(keyOf(merged), merged); + return { id: found.row.id }; + }, + async delete(_t: string, o: { where: Record }) { + assertEngineDeleteDispatch(o); + const found = findRow(o.where); + if (!found) return { deleted: 0 }; + rows.delete(found.key); + return { deleted: 1 }; + }, + async transaction(cb: (ctx: any, info: { owned: boolean }) => Promise): Promise { + return cb(undefined, { owned: true }); + }, + registry: { registerItem: () => {}, registerObject: () => {}, getPackage: () => undefined }, + }; + if (opts.provider !== false) engine.getKeyedDigest = () => keyedDigest; + return { engine, rows, historyRows, auditRows }; +} + +const viewBody = (label: string) => ({ + name: 'case_grid', + type: 'grid', + label, + columns: ['id', 'title'], + object: 'case', + viewKind: 'list', +}); +const ORG = 'org_alpha'; +const ref = { type: 'view', name: 'case_grid', organizationId: ORG, actor: 'admin' } as const; + +/** Every stored content hash the double holds, active rows and history alike. */ +function storedHashes(h: ReturnType): Set { + const out = new Set(); + for (const r of h.rows.values()) if (typeof r.checksum === 'string') out.add(r.checksum); + for (const r of h.historyRows) { + if (typeof r.checksum === 'string') out.add(r.checksum); + if (typeof r.previous_checksum === 'string') out.add(r.previous_checksum); + } + return out; +} + +function activeHash(h: ReturnType): string { + const row = [...h.rows.values()].find((r) => r.name === 'case_grid' && r.state === 'active'); + return String(row?.checksum); +} + +async function rejection(run: () => Promise): Promise { + let caught: any; + let resolved = false; + try { + await run(); + resolved = true; + } catch (e) { + caught = e; + } + expect(resolved, 'expected a refusal, but the call resolved').toBe(false); + return caught; +} + +/** No stored hash, in either the message or any attribute of a refusal. */ +function expectNoStoredHash(value: unknown, stored: Set): void { + const text = typeof value === 'string' ? value : JSON.stringify({ + ...(value as object), + message: (value as { message?: string })?.message, + }); + for (const s of stored) expect(text).not.toContain(s); + expect(text).not.toMatch(SHA256); +} + +describe('[#21207] /meta receipts serve the keyed form of the stored content hash', () => { + it('save receipt: keyed, not the stored hash, not a recomputation; stable; moves on change', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + + const first: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const stored1 = activeHash(h); + expect(first.version).toMatch(KEYED); + expect(first.version).not.toBe(stored1); + expect(first.version).not.toBe(hashSpec(viewBody('v1'))); + expect(first.version).toBe(await keyedDigest(stored1)); + + // An identical re-save writes nothing and serves the same value. + const again: any = await p.saveMetaItem({ ...ref, item: viewBody('v1'), parentVersion: first.version } as any); + expect(again.version).toBe(first.version); + + const changed: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: first.version } as any); + expect(changed.version).toMatch(KEYED); + expect(changed.version).not.toBe(first.version); + expect(changed.version).not.toBe(activeHash(h)); + }); + + it('publish and rollback receipts: keyed, never the stored hash', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + + await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + await p.saveMetaItem({ ...ref, item: viewBody('staged'), mode: 'draft' } as any); + const published: any = await p.publishMetaItem({ ...ref } as any); + expect(published.version).toMatch(KEYED); + expect(published.version).toBe(await keyedDigest(activeHash(h))); + expect(storedHashes(h).has(published.version)).toBe(false); + + const rolled: any = await p.rollbackMetaItem({ ...ref, toVersion: 1 } as any); + expect(rolled.version).toMatch(KEYED); + expect(rolled.version).toBe(await keyedDigest(activeHash(h))); + expect(storedHashes(h).has(rolled.version)).toBe(false); + }); +}); + +describe('[#21207] the history read serves keyed hashes per event', () => { + it('every event hash and parent hash is keyed, none is stored, and two reads agree', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + + const read1 = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + const read2 = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + expect(read1.events.length).toBeGreaterThanOrEqual(2); + for (const ev of read1.events) { + expect(ev.hash).toMatch(KEYED); + if (ev.parentHash !== null) expect(ev.parentHash).toMatch(KEYED); + } + expect(read1.events.some((ev) => ev.parentHash !== null)).toBe(true); + expectNoStoredHash(JSON.stringify(read1), storedHashes(h)); + expect(JSON.stringify(read2)).toBe(JSON.stringify(read1)); + // The head event's keyed hash IS the token the save receipt served. + expect(read1.events.map((e) => e.hash)).toContain(v1.version); + }); +}); + +describe('[#21207] inbound version tokens are compared in keyed form', () => { + it('save door: the served token is accepted, the raw stored hash is refused (METADATA_CONFLICT / 409)', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const raw = activeHash(h); + + const refused = await rejection(() => + p.saveMetaItem({ ...ref, item: viewBody('raw'), parentVersion: raw } as any)); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + expect(activeHash(h)).toBe(raw); + + const accepted: any = await p.saveMetaItem({ ...ref, item: viewBody('keyed'), parentVersion: v1.version } as any); + expect(accepted.success).toBe(true); + expect(activeHash(h)).not.toBe(raw); + }); + + it('save door: a stale keyed token is refused, and the refusal carries the keyed value or none', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const v2: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + + const stale = await rejection(() => + p.saveMetaItem({ ...ref, item: viewBody('lost'), parentVersion: v1.version } as any)); + expect(stale.code).toBe('METADATA_CONFLICT'); + expect(stale.status).toBe(409); + expectNoStoredHash(stale, storedHashes(h)); + for (const attr of [stale.expectedParent, stale.actualHead]) { + if (attr !== undefined && attr !== null) expect(attr).toMatch(KEYED); + } + expect(stale.actualHead).toBe(v2.version); + }); + + it('the decision-audit note of a refused write names no hash, stored or keyed', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + await rejection(() => p.saveMetaItem({ ...ref, item: viewBody('lost'), parentVersion: v1.version } as any)); + + const denials = h.auditRows.filter((a) => a.code === 'metadata_conflict'); // adr0112-ok: D6b persisted audit column + expect(denials).toHaveLength(1); + const note = String(denials[0]!.note); + expect(note).not.toMatch(SHA256); + expect(note).not.toMatch(/hmac-sha256:/); + expectNoStoredHash(note, storedHashes(h)); + }); + + it('reset door: the served token is accepted, the raw stored hash is refused', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + + const refused = await rejection(() => + p.deleteMetaItem({ ...ref, parentVersion: activeHash(h) } as any)); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + expectNoStoredHash(refused, storedHashes(h)); + + const reset: any = await p.deleteMetaItem({ ...ref, parentVersion: v1.version } as any); + expect(reset.success).toBe(true); + }); +}); + +describe('[#21207] no crypto provider: tokens keyed under a process-scoped ephemeral key', () => { + it('receipts and history serve a keyed token: never empty, never stored, distinct per content, stable', async () => { + const h = makeEngine({ provider: false }); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const stored1 = activeHash(h); + expect(v1.version).toMatch(KEYED); + expect(v1.version).not.toBe(stored1); + expect(v1.version).not.toBe(hashSpec(viewBody('v1'))); + // Not this file's test key either: the key is the process's own. + expect(v1.version).not.toBe(await keyedDigest(stored1)); + + const v2: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + expect(v2.version).toMatch(KEYED); + expect(v2.version).not.toBe(v1.version); + + const read1 = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + const read2 = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + expect(read1.events.length).toBeGreaterThanOrEqual(2); + for (const ev of read1.events) expect(ev.hash).toMatch(KEYED); + expect(read1.events.map((e) => e.hash)).toContain(v2.version); + expect(JSON.stringify(read2)).toBe(JSON.stringify(read1)); + expectNoStoredHash(JSON.stringify({ v1, v2, read1 }), storedHashes(h)); + }); + + it('the served token is accepted on both doors; a stale, raw, empty or withheld token is refused (METADATA_CONFLICT / 409)', async () => { + const h = makeEngine({ provider: false }); + const p = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const v2: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + expect(v2.success).toBe(true); + const raw = activeHash(h); + + for (const token of [v1.version, raw, '', '(withheld)']) { + for (const run of [ + () => p.saveMetaItem({ ...ref, item: viewBody('lost'), parentVersion: token } as any), + () => p.deleteMetaItem({ ...ref, parentVersion: token } as any), + ]) { + const refused = await rejection(run); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + expectNoStoredHash(refused, storedHashes(h)); + } + } + expect(activeHash(h)).toBe(raw); + + const reset: any = await p.deleteMetaItem({ ...ref, parentVersion: v2.version } as any); + expect(reset.success).toBe(true); + }); + + it('one key per process: a second protocol answers the first one\'s token', async () => { + const h = makeEngine({ provider: false }); + const first = new ObjectStackProtocolImplementation(h.engine); + const second = new ObjectStackProtocolImplementation(h.engine); + const v1: any = await first.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const v2: any = await second.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: v1.version } as any); + expect(v2.success).toBe(true); + }); + + it('a provider registered later moves the tokens: the held token is refused once, the next served one is accepted', async () => { + const h = makeEngine({ provider: false }); + const p = new ObjectStackProtocolImplementation(h.engine); + const before: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + + h.engine.getKeyedDigest = () => keyedDigest; + const refused = await rejection(() => + p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: before.version } as any)); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + + const { events } = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + const current = events.find((e) => e.hash === refused.actualHead); + expect(refused.actualHead).toBe(await keyedDigest(activeHash(h))); + expect(current).toBeDefined(); + const after: any = await p.saveMetaItem({ ...ref, item: viewBody('v2'), parentVersion: refused.actualHead } as any); + expect(after.success).toBe(true); + expect(after.version).toBe(await keyedDigest(activeHash(h))); + }); +}); + +describe('[#21207] a sent token is never read as no pin', () => { + it('an empty or withheld token is refused on both doors with a provider registered (METADATA_CONFLICT / 409)', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const raw = activeHash(h); + for (const token of ['', '(withheld)']) { + for (const run of [ + () => p.saveMetaItem({ ...ref, item: viewBody('lost'), parentVersion: token } as any), + () => p.deleteMetaItem({ ...ref, parentVersion: token } as any), + ]) { + const refused = await rejection(run); + expect(refused.code).toBe('METADATA_CONFLICT'); + expect(refused.status).toBe(409); + } + } + expect(activeHash(h)).toBe(raw); + }); +}); + +describe('[#21207] a change note that quotes a stored hash', () => { + it('the publish door writes a note that quotes no hash', async () => { + const h = makeEngine(); + const p = new ObjectStackProtocolImplementation(h.engine); + await p.saveMetaItem({ ...ref, item: viewBody('staged'), mode: 'draft' } as any); + await p.publishMetaItem({ ...ref } as any); + const notes = h.historyRows.map((r) => r.change_note).filter((n) => typeof n === 'string') as string[]; + expect(notes.length).toBeGreaterThan(0); + for (const note of notes) expect(note).not.toMatch(SHA256); + }); + + it('the history read serves a stored note\'s quoted hash keyed, under the provider\'s key or the process key', async () => { + for (const provider of [true, false]) { + const h = makeEngine({ provider }); + const p = new ObjectStackProtocolImplementation(h.engine); + const saved: any = await p.saveMetaItem({ ...ref, item: viewBody('v1') } as any); + const stored = activeHash(h); + // A row written before the publish door stated its own message. + for (const row of h.historyRows) row.change_note = `publish draft (hash ${stored})`; + + const { events } = await p.historyMetaItem({ type: 'view', name: 'case_grid', organizationId: ORG }); + expect(events.length).toBeGreaterThan(0); + for (const ev of events) { + // The quote is served as the very token the receipt served. + expect(ev.message).toBe(`publish draft (hash ${saved.version})`); + if (provider) expect(saved.version).toBe(await keyedDigest(stored)); + } + expectNoStoredHash(JSON.stringify(events), storedHashes(h)); + } + }); +}); diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 237a0530d7a..f95762156e2 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -192,6 +192,18 @@ import { storedMetadataBodyGroupingRefusal, storedMetadataBodyPredicateRefusal, storedMetadataBodyProjection, + // [#21207] The stored content hash of the same rows: served keyed, never + // evaluated — see `STORED_METADATA_HASH_COLUMNS`. + ephemeralStoredHashDigest, + isStoredMetadataBodyObject, + servedContentHash, + serveStoredHashTokens, + serveStoredMetadataHashColumnRows, + serveStoredMetadataHashColumns, + STORED_METADATA_UNSEARCHABLE_COLUMNS, + storedMetadataHashEvaluateRefusal, + storedMetadataSearchRefusal, + type StoredHashDigest, } from './metadata-redaction.js'; import type { StoredFlowCanonicalization, @@ -2602,6 +2614,28 @@ function declaresClientRefusal(err: unknown): boolean { return typeof status === 'number' && status >= 400 && status < 500; } +/** + * [#21207] A caller's version token that names no current stored head, found by + * the protocol's own KEYED comparison before the repository is asked (see + * {@link ObjectStackProtocolImplementation.storedParentForToken}). + * + * A `ConflictError`, so each door's existing conflict branch — the 409 + * `METADATA_CONFLICT` and its decision-audit row — handles it unchanged; told + * apart from the repository's own race conflict because its `expectedParent` + * is the CALLER's token rather than a stored hash. The message is replaced: + * the base class prints both values, and one of them is the stored hash. + */ +class InboundVersionConflictError extends ConflictError { + constructor( + ref: { org: string; type: string; name: string }, + token: string, + currentStored: string | null, + ) { + super(ref as ConstructorParameters[0], token, currentStored); + this.message = `Conflict on ${ref.type}/${ref.name}: the version token sent is not the current version`; + } +} + /** * [#8136] The client-facing sentence for a failed overlay delete: the caller's * own refusal when they declared one, and otherwise a stable line that names @@ -11146,6 +11180,62 @@ export class ObjectStackProtocolImplementation implements throw err; } + /** + * [#21207] A `search` on a stored-metadata table never scans its body or + * content-hash columns ({@link STORED_METADATA_UNSEARCHABLE_COLUMNS}). + * + * A search is a substring filter the engine evaluates over every column it + * scans, and with no `searchableFields` declared it scans every text-like + * column — the stored body and both stored hashes among them. Over those + * it is the verifier the filter refusals close: a guessed hash, or a guessed + * prefix of withheld credential material, returns the row exactly when it + * is right. So an explicit field list naming one is refused + * (`INVALID_FIELD` / 400, the evaluate refusals' envelope), and a search + * that names none is handed to the engine with the object's searchable set + * minus those columns — the engine intersects an override with that set and + * never widens it. Runs after {@link assertSearchFieldsAreSearchable}, so a + * name that is not searchable at all keeps its own answer. + */ + private narrowStoredMetadataSearch( + object: string, + options: Record, + wireSpelling: Record, + ): void { + if (!isStoredMetadataBodyObject(object)) return; + const objectForm = options.search !== null && typeof options.search === 'object'; + const [explicit, param] = options.searchFields != null + ? [options.searchFields, wireSpelling.searchFields ?? 'searchFields'] + : objectForm && options.search.fields != null + ? [options.search.fields, wireSpelling.search ?? 'search'] + : [undefined, '']; + const names: string[] = typeof explicit === 'string' + ? explicit.split(',').map((s: string) => s.trim()).filter(Boolean) + : Array.isArray(explicit) ? explicit.filter((f: unknown): f is string => typeof f === 'string') : []; + if (names.length > 0) { + const refusal = storedMetadataSearchRefusal(object, names, param); + if (refusal) throw refusal; + return; + } + if (options.search == null) return; + const gate = this.resolveQueryFields(object); + // No field map: the engine has none to expand a search over either. + if (!gate) return; + const { allowed } = resolveSearchFieldResolution({ + fields: gate.fields, + searchableFields: gate.schema?.searchableFields, + displayField: gate.schema?.nameField ?? gate.schema?.displayNameField, + }); + const narrowed = allowed.filter((field) => !STORED_METADATA_UNSEARCHABLE_COLUMNS.includes(field)); + if (narrowed.length === 0) { + // An empty override is ABSENT to the engine, which would then scan + // the whole default set — these columns included. Refuse instead. + const refusal = storedMetadataSearchRefusal(object, allowed, wireSpelling.search ?? 'search'); + if (refusal) throw refusal; + return; + } + options.searchFields = narrowed; + } + /** * [#4254] GROUP-BY axis. A grouping target the object does not have is * refused (`400 INVALID_FIELD`); a grouping target the spec cannot read is @@ -11674,6 +11764,9 @@ export class ObjectStackProtocolImplementation implements request.object, (options.search as any).fields, wireSpelling.search ?? 'search', ); } + // [#21207] …and on a stored-metadata table a search never scans the body + // or content-hash columns: refused when named, narrowed away otherwise. + this.narrowStoredMetadataSearch(request.object, options, wireSpelling); // Boolean fields for (const key of ['distinct', 'count']) { @@ -11799,13 +11892,22 @@ export class ObjectStackProtocolImplementation implements ? (options.aggregations as ReadonlyArray<{ filter?: unknown }>).flatMap((a) => collectFilterFieldKeys(a?.filter)) : []; - const bodyPredicateRefusal = storedMetadataBodyPredicateRefusal(request.object, { - filterFields: [...collectFilterFieldKeys(options.where), ...aggregationFilterFields], - sortFields: Array.isArray(options.orderBy) - ? (options.orderBy as ReadonlyArray<{ field?: unknown }>).map((e) => e?.field) - : [], - }); + const filterFields = [...collectFilterFieldKeys(options.where), ...aggregationFilterFields]; + const sortFields = Array.isArray(options.orderBy) + ? (options.orderBy as ReadonlyArray<{ field?: unknown }>).map((e) => e?.field) + : []; + const bodyPredicateRefusal = storedMetadataBodyPredicateRefusal(request.object, { filterFields, sortFields }); if (bodyPredicateRefusal) throw bodyPredicateRefusal; + // [#21207] The same three shapes on the stored CONTENT-HASH columns + // (maintainer ruling A on the second execution fork): a group key would + // serve the stored hash, a filter on it is an online verifier, a sort + // orders by it. Refused in the body column's envelope, before the engine. + const hashEvaluateRefusal = storedMetadataHashEvaluateRefusal(request.object, { + groupBy: options.groupBy, + filterFields, + sortFields, + }); + if (hashEvaluateRefusal) throw hashEvaluateRefusal; // Route to engine.aggregate() when the query has GROUP BY / aggregations. // engine.find() does not do in-memory aggregation fallback, so without @@ -11903,10 +12005,18 @@ export class ObjectStackProtocolImplementation implements // projection named only the body, and taken back off before serving. const bodyProjection = storedMetadataBodyProjection(request.object, options.fields); if (bodyProjection.addedType) options.fields = bodyProjection.fields; - const records = redactStoredMetadataRows( + // [#21207] …and its stored CONTENT HASH (`checksum`, `previous_checksum`) + // is served in keyed form — never the stored value, which beside the + // projected body confirms a guess at the withheld material offline — + // under the provider's key, or the process-scoped ephemeral one. + const records = await serveStoredMetadataHashColumnRows( request.object, - await this.engine.find(request.object, options), - { dropType: bodyProjection.addedType }, + redactStoredMetadataRows( + request.object, + await this.engine.find(request.object, options), + { dropType: bodyProjection.addedType }, + ), + this.storedHashDigest(), ); // Pagination metadata. When a `limit` is present the response is a single // page, so `records.length` is the page size — NOT the match total. Run a @@ -12031,7 +12141,13 @@ export class ObjectStackProtocolImplementation implements return { object: request.object, id: request.id, - record: redactStoredMetadataRow(request.object, result, { dropType: bodyProjection.addedType }), + // [#21207] Same served form as the list path: the content-hash + // columns keyed. + record: await serveStoredMetadataHashColumns( + request.object, + redactStoredMetadataRow(request.object, result, { dropType: bodyProjection.addedType }), + this.storedHashDigest(), + ), }; } throw recordNotFoundError(request.object, request.id); @@ -15669,6 +15785,101 @@ export class ObjectStackProtocolImplementation implements await this.recordMetadataAudit(ObjectStackProtocolImplementation.optimisticConflictAuditEntry(args)); } + // ----------------------------------------------------------------------- + // [#21207] The stored content hash, as the `/meta` doors serve and take it + // ----------------------------------------------------------------------- + // + // Maintainer ruling B on #21207: the stored content hash of a metadata body + // stays the canonical SHA-256 at rest — the repository contract, its + // producers and the parent links are untouched — but no door hands it out. + // It is a hash over the WHOLE stored body, withheld credential material + // included, so beside the projected body it confirms a guess at that + // material offline. Every door that serves it serves a keyed digest of it + // (the crypto provider's, or with none registered a process-scoped + // ephemeral key's); every door that takes a version token back compares the + // token in that same form and hands the STORED value to the repository. + + /** + * The keyed digest the doors serve and compare under: the registered crypto + * provider's, read from the engine at the moment of use — a host registers + * the provider AFTER the kernel starts, so a value read once at + * construction would answer "none" for good — and, while none is registered + * (or the host engine has no such accessor), {@link ephemeralStoredHashDigest}. + * + * ⛔ Never `undefined`: a door with no key would serve no token, every + * caller would then hold the same empty one, and a client that sends no pin + * for an empty token would turn every pinned write into an unpinned one — + * the optimistic lock failing OPEN. + */ + private storedHashDigest(): StoredHashDigest { + const accessor = this.engine?.getKeyedDigest; + const provider: StoredHashDigest | undefined = + typeof accessor === 'function' ? accessor.call(this.engine) : undefined; + return provider ?? ephemeralStoredHashDigest; + } + + /** + * A write receipt's `version` — the version token a caller sends back as + * `If-Match` — for a write whose stored content hash is `stored`: its keyed + * digest ({@link storedHashDigest}). Never empty, never the stored value. + */ + private async receiptVersion(stored: string): Promise { + return this.storedHashDigest()(stored); + } + + /** + * Resolve a caller's version token to the STORED head it names, for a write + * whose current stored head is `currentStored`: the token must equal the + * keyed digest of that head, and the stored value is what the repository's + * own optimistic lock then compares. `null` keeps its meaning ("expect no + * row") and carries no hash. Anything else — the raw stored hash a pre-keying + * client still holds, a stale token, an empty or withheld token, a token + * keyed before a restart or before a provider was registered — is an + * {@link InboundVersionConflictError}, which the door's conflict branch + * answers 409. ⛔ A sent token is never read as "no pin". + */ + private async storedParentForToken( + ref: { org: string; type: string; name: string }, + token: string | null, + currentStored: string | null, + ): Promise { + if (token === null) return null; + if (currentStored !== null && token !== '' && (await this.storedHashDigest()(currentStored)) === token) { + return currentStored; + } + throw new InboundVersionConflictError(ref, token, currentStored); + } + + /** + * The 409 `METADATA_CONFLICT` a door answers for a `ConflictError`, whose + * text and attributes carry the SERVED (keyed) form of a stored hash or + * none at all — never a stored hash. `subject` names the item, `prefix` is + * the door's own first sentence. + * + * - a repository race (the stored head moved between the door's read and + * its write): `Expected parent X but current is Y`, both keyed; + * - a caller's token naming no current head: the keyed current head. + * + * A side with no served form is `(withheld)`; an absent side is `null`. + */ + private async metadataConflictRefusal(err: ConflictError, subject: string, prefix: string): Promise { + const conflict: any = new Error(subject); + conflict.code = 'METADATA_CONFLICT'; + conflict.status = 409; + const digest = this.storedHashDigest(); + const show = (served: string | null | undefined) => (served === undefined ? '(withheld)' : served ?? 'null'); + const current = await servedContentHash(err.actualHead, digest); + if (current !== undefined) conflict.actualHead = current; + if (err instanceof InboundVersionConflictError) { + conflict.message = `${prefix} The version token sent is not the current version (current is ${show(current)}).`; + return conflict; + } + const expected = await servedContentHash(err.expectedParent, digest); + if (expected !== undefined) conflict.expectedParent = expected; + conflict.message = `${prefix} Expected parent ${show(expected)} but current is ${show(current)}.`; + return conflict; + } + /** * [#8594] The same row as a VALUE, for the site that must not write it * where the conflict is caught — see {@link lockWriteRefusal} for the full @@ -15698,7 +15909,15 @@ export class ObjectStackProtocolImplementation implements ...(args.actor ? { actor: args.actor } : {}), source: args.source, ...(args.requestId ? { requestId: args.requestId } : {}), - note: `expected parent ${args.expectedParent ?? 'null'} but current is ${args.actualHead ?? 'null'}`, + // [#21207] Fork three, ruling A: a COPY never carries the stored + // content hash — not raw (an offline verifier, at rest and served to + // every audit reader), and not keyed either (a copy has no use for a + // version token, and a keyed value would die with the key). The + // note keeps its sentence and says which side was absent; a value + // is `(withheld)`. `os migrate audit-metadata-bodies` rewrites the + // notes written before this to exactly this text. + note: `expected parent ${args.expectedParent == null ? 'null' : '(withheld)'} ` + + `but current is ${args.actualHead == null ? 'null' : '(withheld)'}`, }; } @@ -17042,7 +17261,13 @@ export class ObjectStackProtocolImplementation implements } } - async saveMetaItem(request: { type: string, name: string, item?: any, organizationId?: string, parentVersion?: string | null, actor?: string, force?: boolean, mode?: 'draft' | 'publish', packageId?: string | null, source?: string, writeFace?: MetadataWriteFace }) { + // [#21207] `parentVersion` is a CALLER's version token — the keyed form a + // receipt served — and is compared in that form (`storedParentForToken`). + // `storedParentVersion` is the in-process twin for a caller that read the + // STORED content hash itself (`migrateStoredMetadata`): it reaches the + // repository as given. ⛔ No transport sets it — every door builds its + // request field by field from named inputs, never by spreading a body. + async saveMetaItem(request: { type: string, name: string, item?: any, organizationId?: string, parentVersion?: string | null, storedParentVersion?: string | null, actor?: string, force?: boolean, mode?: 'draft' | 'publish', packageId?: string | null, source?: string, writeFace?: MetadataWriteFace }) { // [commit fd6bdf89f] The ADR-0112 envelope this refusal always owed. Every OTHER // refusal in this method declares `code` AND `status` // (`NOT_OVERRIDABLE`/403, `NOT_CREATABLE`/403, `ITEM_LOCKED`/403, @@ -17914,8 +18139,9 @@ export class ObjectStackProtocolImplementation implements org: orgId ?? 'env', } as Parameters[0]; let parentVersion: string | null; - if (request.parentVersion !== undefined) { - parentVersion = request.parentVersion; + if (request.storedParentVersion !== undefined) { + // [#21207] An in-process caller that read the stored hash itself. + parentVersion = request.storedParentVersion; } else { // Parent is scoped to the lifecycle we're about to write: // a draft's parent is the current draft hash (or null @@ -17927,7 +18153,22 @@ export class ObjectStackProtocolImplementation implements state: mode === 'draft' ? 'draft' : 'active', packageId: request.packageId ?? null, }); - parentVersion = current?.hash ?? null; + const currentStored = current?.hash ?? null; + if (request.parentVersion === undefined) { + parentVersion = currentStored; + } else { + // [#21207] A caller's version token names the stored head only + // in keyed form; the repository's own lock then runs on the + // stored value. A token naming no current head — the raw + // stored hash included — is answered here, in this door's own + // conflict envelope and with its own audit row. + try { + parentVersion = await this.storedParentForToken(ref, request.parentVersion, currentStored); + } catch (err: unknown) { + if (err instanceof ConflictError) throw await this.saveConflict(err, request, orgId, writeSource); + throw err; + } + } } // [#8154] THE WRITE-PATH INVERSE of the read exits' credential // redaction — the half without which this card's fix is a DATA-LOSS @@ -18056,7 +18297,9 @@ export class ObjectStackProtocolImplementation implements }); return { success: true, - version: result.version, + // [#21207] The version token: the keyed form of the stored + // content hash, never the stored value (see `receiptVersion`). + version: await this.receiptVersion(result.version), seq: result.seq, ...(projectionApplied ? { projectionApplied } : {}), // [#4717] #4463 D3's advisory half, finally on the response. @@ -18116,31 +18359,42 @@ export class ObjectStackProtocolImplementation implements : `Saved ${singularTypeForRepo} '${request.name}' (env-wide, state=${mode === 'draft' ? 'draft' : 'active'}) [seq=${result.seq}]`), }; } catch (err: any) { - if (err instanceof ConflictError) { - const conflict = new Error( - `${request.type}/${request.name} has been modified since you loaded it. ` - + `Expected parent ${err.expectedParent ?? 'null'} but current is ${err.actualHead ?? 'null'}.`, - ); - (conflict as any).code = 'METADATA_CONFLICT'; - (conflict as any).status = 409; - (conflict as any).expectedParent = err.expectedParent; - (conflict as any).actualHead = err.actualHead; - await this.recordOptimisticConflictAudit({ - type: request.type, - name: request.name, - organizationId: orgId, - operation: 'save', - ...(request.actor ? { actor: request.actor } : {}), - source: writeSource, - expectedParent: err.expectedParent, - actualHead: err.actualHead, - }); - throw conflict; - } + if (err instanceof ConflictError) throw await this.saveConflict(err, request, orgId, writeSource); throw err; } } + /** + * The save door's 409 for a `ConflictError` — the repository's race, or a + * caller's token the keyed comparison refused — plus its decision-audit row. + * One builder for both, so the two cannot answer in different words. + * [#21207] The text and attributes carry keyed values or none + * ({@link metadataConflictRefusal}); the audit row carries no hash at all. + */ + private async saveConflict( + err: ConflictError, + request: { type: string; name: string; actor?: string }, + orgId: string | null, + writeSource: string, + ): Promise { + const conflict = await this.metadataConflictRefusal( + err, + `${request.type}/${request.name}`, + `${request.type}/${request.name} has been modified since you loaded it.`, + ); + await this.recordOptimisticConflictAudit({ + type: request.type, + name: request.name, + organizationId: orgId, + operation: 'save', + ...(request.actor ? { actor: request.actor } : {}), + source: writeSource, + expectedParent: err.expectedParent, + actualHead: err.actualHead, + }); + return conflict; + } + /** * `os migrate meta --stored` — canonicalize `sys_metadata` rows in place so * the read-path conversion chain has a finish line (#4327). @@ -18577,7 +18831,10 @@ export class ObjectStackProtocolImplementation implements name: base.name, item, mode: state === 'draft' ? 'draft' : 'publish', - parentVersion: row.checksum ?? null, + // [#21207] The STORED hash this pass read itself — the + // in-process spelling, never compared in keyed form (the + // pass already holds the stored value; nothing to key). + storedParentVersion: row.checksum ?? null, packageId, force: true, source: 'migrate-stored', @@ -18693,7 +18950,21 @@ export class ObjectStackProtocolImplementation implements const opts: { sinceSeq?: number; limit?: number } = {}; if (request.sinceSeq !== undefined) opts.sinceSeq = request.sinceSeq; if (request.limit !== undefined) opts.limit = request.limit; - for await (const ev of repo.history(ref, opts)) events.push(ev); + // [#21207] Each event's hash and parent hash are served in keyed form — + // the same value the write receipts hand out, so an event still names + // the token a caller holds; a delete event's own `null` is kept. + // The event's message is the row's change note, which can QUOTE a stored + // hash (`publish draft (hash …)` on rows written before the publish door + // stated its own message) — each quote is served the same way. + const digest = this.storedHashDigest(); + for await (const ev of repo.history(ref, opts)) { + events.push({ + ...ev, + hash: (await servedContentHash(ev.hash, digest)) ?? null, + parentHash: (await servedContentHash(ev.parentHash, digest)) ?? null, + ...(typeof ev.message === 'string' ? { message: await serveStoredHashTokens(ev.message, digest) } : {}), + }); + } return { events }; } @@ -18965,7 +19236,8 @@ export class ObjectStackProtocolImplementation implements advisories?: RuntimeAuthoringIssue[]; } = { success: true, - version: result.version, + // [#21207] Keyed, never the stored content hash (`receiptVersion`). + version: await this.receiptVersion(result.version), seq: result.seq, message: `Published draft — type=${request.type}, name=${request.name} [seq=${result.seq}]`, // [#9176] Omitted-when-empty, never `advisories: []` — a clean @@ -19226,7 +19498,12 @@ export class ObjectStackProtocolImplementation implements // #4556 — NULL, not 'system', for an actor-less publish. actor: request.actor ?? null, source: 'protocol.publishMetaItem', - ...(request.message ? { message: request.message } : {}), + // [#21207] Always a message of the caller's or this door's own: + // left unstated, the repository records `publish draft (hash …)`, + // quoting the draft's stored content hash into the history row's + // change note — served to every history reader and copied by the + // audit writer. This door's default says what happened without it. + message: request.message || 'publish draft', intent, // [#8907] Spread, not `packageId: request.packageId`: `null` is // a meaningful scope (the unbound row) and `undefined` means @@ -19237,14 +19514,12 @@ export class ObjectStackProtocolImplementation implements return { singularType, orgId, advisories: runtimeAdvisories, result }; } catch (err: any) { if (err instanceof ConflictError) { - const conflict: any = new Error( - `${request.type}/${request.name} published row advanced while you held the draft. ` - + `Expected parent ${err.expectedParent ?? 'null'} but current is ${err.actualHead ?? 'null'}.`, + // [#21207] Keyed values or none in the text and attributes. + const conflict = await this.metadataConflictRefusal( + err, + `${request.type}/${request.name}`, + `${request.type}/${request.name} published row advanced while you held the draft.`, ); - conflict.code = 'METADATA_CONFLICT'; - conflict.status = 409; - conflict.expectedParent = err.expectedParent; - conflict.actualHead = err.actualHead; // [#8594] Attached, not written — same reason as the lock gate // above. The repository's own transaction has already unwound by // the time this `catch` runs, but the BATCH caller's has not. @@ -20507,7 +20782,10 @@ export class ObjectStackProtocolImplementation implements // element's bytes are unchanged, and absence means "nothing to // report", never "the gate did not run". published.push({ - type: p.d.type, name: p.d.name, version: p.version, + // [#21207] Each element's version token is keyed, like the + // single-item doors' (`receiptVersion`); `p.version` stays the + // stored hash for everything internal. + type: p.d.type, name: p.d.name, version: await this.receiptVersion(p.version), ...(p.advisories.length > 0 ? { advisories: p.advisories } : {}), }); try { @@ -22784,21 +23062,20 @@ export class ObjectStackProtocolImplementation implements }); return { success: true, - version: result.version, + // [#21207] Keyed, never the stored content hash (`receiptVersion`). + version: await this.receiptVersion(result.version), seq: result.seq, restoredFromVersion: request.toVersion, message: `Reverted to version ${request.toVersion} — type=${request.type}, name=${request.name} [seq=${result.seq}]`, }; } catch (err: any) { if (err instanceof ConflictError) { - const conflict: any = new Error( - `${request.type}/${request.name} advanced during rollback. ` - + `Expected parent ${err.expectedParent ?? 'null'} but current is ${err.actualHead ?? 'null'}.`, + // [#21207] Keyed values or none in the text and attributes. + const conflict = await this.metadataConflictRefusal( + err, + `${request.type}/${request.name}`, + `${request.type}/${request.name} advanced during rollback.`, ); - conflict.code = 'METADATA_CONFLICT'; - conflict.status = 409; - conflict.expectedParent = err.expectedParent; - conflict.actualHead = err.actualHead; await this.recordOptimisticConflictAudit({ type: request.type, name: request.name, @@ -23311,8 +23588,13 @@ export class ObjectStackProtocolImplementation implements // Last-write-wins parent resolution unless the caller pinned // an explicit version (Studio's "Reset" button is unpinned; // a future "delete vN" flow can pass parentVersion). - const parentVersion: string = request.parentVersion !== undefined - ? (request.parentVersion ?? current.hash) + // [#21207] A pinned version is a caller's token, compared in + // keyed form against the current stored head + // (`storedParentForToken`); the repository's lock then runs on + // the stored value. A token naming no current head throws a + // `ConflictError`, answered by the conflict branch below. + const parentVersion: string = typeof request.parentVersion === 'string' + ? ((await this.storedParentForToken(ref, request.parentVersion, current.hash)) ?? current.hash) : current.hash; const result = await repo.delete(ref, { @@ -23413,14 +23695,12 @@ export class ObjectStackProtocolImplementation implements }; } catch (err: any) { if (err instanceof ConflictError) { - const conflict = new Error( - `${request.type}/${request.name} has been modified since you loaded it. ` - + `Expected parent ${err.expectedParent ?? 'null'} but current is ${err.actualHead ?? 'null'}.`, + // [#21207] Keyed values or none in the text and attributes. + const conflict = await this.metadataConflictRefusal( + err, + `${request.type}/${request.name}`, + `${request.type}/${request.name} has been modified since you loaded it.`, ); - (conflict as any).code = 'METADATA_CONFLICT'; - (conflict as any).status = 409; - (conflict as any).expectedParent = err.expectedParent; - (conflict as any).actualHead = err.actualHead; await this.recordOptimisticConflictAudit({ type: request.type, name: request.name, diff --git a/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts b/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts index b9878da482a..a4769ff2d97 100644 --- a/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts +++ b/packages/metadata-protocol/src/stored-metadata-body-family.pin.test.ts @@ -38,6 +38,17 @@ * - the generic data door reads / groupBy / filter+sort → this package * (`protocol.data-door-stored-metadata-redaction.test.ts`, and the behaviour * asserted below). + * + * [#21207] Exit two — the same rows' stored CONTENT HASH (`checksum`, and the + * history table's `previous_checksum`), a hash over the whole stored body, + * withheld credential material included. Every surface that SERVES it serves + * a keyed digest (`keyed`: the crypto provider's, or a process-scoped ephemeral + * key's while none is registered), every surface that takes a version token + * back compares it keyed, every COPY drops it (`withheld`), and + * every EVALUATE shape refuses — enumerated as rows below, each with its pin, + * plus a third tooth: the hash columns the object definitions declare are + * exactly `STORED_METADATA_HASH_COLUMNS`, so a new hash-like column fails here + * instead of being served raw. */ import { describe, expect, it } from 'vitest'; @@ -49,8 +60,10 @@ import { } from '@objectstack/spec/kernel'; import { redactStoredMetadataRow, + STORED_METADATA_HASH_COLUMNS, storedMetadataBodyGroupingRefusal, storedMetadataBodyPredicateRefusal, + storedMetadataHashEvaluateRefusal, } from './metadata-redaction.js'; /** The generic data-door verbs this family's seam covers, and how. */ @@ -77,6 +90,40 @@ const FAMILY_SURFACES = [ disposition: 'refuses', pin: '@objectstack/mcp', }, + // [#21207] Exit two: the stored content hash of the same rows. + { + surface: '/meta save, publish, batch-publish and rollback receipts: the version token', + disposition: 'keyed', + pin: 'this package (protocol.served-content-hash.test.ts)', + }, + { surface: '/meta history read: each event\'s hash and parent hash', disposition: 'keyed', pin: 'this package (protocol.served-content-hash.test.ts)' }, + { surface: '/meta save and reset doors: the inbound version token', disposition: 'keyed', pin: 'this package (protocol.served-content-hash.test.ts)' }, + { surface: '409 conflict refusal: its text and attributes', disposition: 'keyed', pin: 'this package (protocol.served-content-hash.test.ts)' }, + { surface: 'decision-audit note of a conflict refusal', disposition: 'withheld', pin: 'this package (protocol.served-content-hash.test.ts)' }, + { surface: 'data door get / list: the content-hash columns', disposition: 'keyed', pin: 'this package (protocol.data-door-stored-content-hash.test.ts)' }, + { surface: 'data door: group / filter / sort on a content-hash column', disposition: 'refuses', pin: 'this package (protocol.data-door-stored-content-hash.test.ts)' }, + { surface: 'data door: search over the body column or a content-hash column', disposition: 'refuses', pin: 'this package (protocol.data-door-stored-content-hash.test.ts)' }, + { surface: 'MCP stdio engine-only reader: the content-hash columns on query / get / the record resource', disposition: 'keyed', pin: '@objectstack/mcp' }, + { surface: 'MCP stdio engine-only reader: group / filter / sort on a content-hash column', disposition: 'refuses', pin: '@objectstack/mcp' }, + { surface: 'audit / activity copy at write time: the content-hash columns', disposition: 'withheld', pin: '@objectstack/plugin-audit' }, + { surface: 'analytics members on a content-hash column', disposition: 'refuses', pin: '@objectstack/service-analytics' }, + { + surface: 'copies at rest (ledger snapshot and diff, activity copy, decision-audit note): os migrate audit-metadata-bodies', + disposition: 'withheld', + pin: '@objectstack/plugin-audit', + }, + // A stored row's change note can QUOTE a stored hash (a draft promotion with + // no message of its own recorded one); found by this card's measurement. + { + surface: 'history change note quoting a stored hash: /meta history message, data door, MCP stdio reader', + disposition: 'keyed', + pin: 'this package (protocol.served-content-hash.test.ts, protocol.data-door-stored-content-hash.test.ts) + @objectstack/mcp', + }, + { + surface: 'history change note: group / filter / sort / search, and as an analytics member', + disposition: 'refuses', + pin: 'this package (protocol.data-door-stored-content-hash.test.ts) + @objectstack/mcp + @objectstack/service-analytics', + }, ] as const; /** A stored datasource body as it sits in the `metadata` column: serialized JSON with credential material. */ @@ -139,14 +186,16 @@ describe('[#21120] stored-metadata-body family — the data door exposes only co } }); - it('enumerates every family surface with a disposition (seam | refuses) and an owning pin', () => { + it('enumerates every family surface with a disposition (seam | keyed | withheld | refuses) and an owning pin', () => { for (const s of FAMILY_SURFACES) { - expect(['seam', 'refuses']).toContain(s.disposition); + expect(['seam', 'keyed', 'withheld', 'refuses']).toContain(s.disposition); expect(s.pin.length).toBeGreaterThan(0); } // One row per the three local data-door surfaces + five cross-package ones - // (audit, analytics, realtime, and the MCP stdio reader's seam and refusal). - expect(FAMILY_SURFACES).toHaveLength(8); + // (audit, analytics, realtime, and the MCP stdio reader's seam and refusal), + // + [#21207] the thirteen surfaces of the stored content hash, and the two + // of the history change note that can quote one. + expect(FAMILY_SURFACES).toHaveLength(23); }); }); @@ -187,3 +236,26 @@ describe('[#21120] stored-metadata-body family — the local surfaces behave', ( expect(storedMetadataBodyPredicateRefusal('showcase_task', { filterFields: ['metadata'] })).toBeUndefined(); }); }); + +describe('[#21207] stored-metadata-body family — the stored content-hash columns', () => { + it('the hash columns the two object definitions declare are exactly STORED_METADATA_HASH_COLUMNS', () => { + const declared = (def: any) => Object.keys(def.fields ?? {}).filter((f) => /checksum|hash/i.test(f)).sort(); + expect(declared(SysMetadataObject)).toEqual(['checksum']); + expect(declared(SysMetadataHistoryObject)).toEqual(['checksum', 'previous_checksum']); + expect([...STORED_METADATA_HASH_COLUMNS].sort()).toEqual(['checksum', 'previous_checksum']); + }); + + it('data-door group / filter / sort on a hash column refuse (INVALID_FIELD / 400); a scalar column does not', () => { + for (const [object, column] of [['sys_metadata', 'checksum'], ['sys_metadata_history', 'previous_checksum']]) { + for (const opts of [{ groupBy: [column] }, { filterFields: [column] }, { sortFields: [column] }]) { + const err = storedMetadataHashEvaluateRefusal(object!, opts) as any; + expect(err?.code).toBe('INVALID_FIELD'); + expect(err?.status).toBe(400); + expect(err?.field).toBe(column); + } + } + expect(storedMetadataHashEvaluateRefusal('sys_metadata', { groupBy: ['type'], filterFields: ['name'], sortFields: ['state'] })) + .toBeUndefined(); + expect(storedMetadataHashEvaluateRefusal('file_blob', { filterFields: ['checksum'] })).toBeUndefined(); + }); +}); diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index eaca19c321a..a2598fb6bbc 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -8562,6 +8562,26 @@ export class ObjectQL implements IObjectQLEngine { } } + /** + * [#21207] Read accessor for the registered provider's keyed digest + * (`ICryptoProvider.keyedDigest`), or `undefined` while no provider is + * registered. + * + * The ONE way a consumer outside this engine reaches the server-held key: + * the doors that serve a stored metadata content hash serve + * `keyedDigest(stored)` instead, and compare a caller's version token in that + * same form. Deliberately narrower than the provider itself — a consumer + * that needs a keyed digest gets that one primitive, never `decrypt`. + * + * Read at the moment of use, never cached by the caller: a host injects the + * provider AFTER the kernel starts (see {@link setCryptoProvider}), so a + * value captured at boot would still say "none" once one is registered. + */ + getKeyedDigest(): ((plain: string) => Promise) | undefined { + const provider = this.cryptoProvider; + return provider ? (plain: string) => provider.keyedDigest(plain) : undefined; + } + /** * [#8022] Observe crypto-provider registration. * diff --git a/packages/plugins/plugin-audit/src/audit-writers.test.ts b/packages/plugins/plugin-audit/src/audit-writers.test.ts index 919ec5fb60e..868cfcfb1cf 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.test.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.test.ts @@ -2273,3 +2273,124 @@ describe('[#21120] stored metadata body copies are redacted at write time', () = expect(JSON.parse(audit!.row.new_value).metadata).toBe(clean); }); }); + +/** + * [#21207] Exit two, fork three: a COPY never carries the stored content hash. + * + * The ledger snapshot and diff, and the activity copy, of a `sys_metadata` / + * `sys_metadata_history` write used to copy the row's `checksum` (and the + * history row's `previous_checksum`) whole — a hash over the stored body, + * withheld credential material included, i.e. an offline verifier, at rest and + * served to every ledger reader. The copy now drops both columns; the history + * table itself remains the lineage. Every other column, and every other + * object's `checksum`, is copied as before. + */ +describe('[#21207] stored metadata copies carry no content hash', () => { + const HASH = `sha256:${'a'.repeat(64)}`; + const PARENT = `sha256:${'b'.repeat(64)}`; + const NEXT = `sha256:${'c'.repeat(64)}`; + const SCHEMAS = { + ...SINGLE_TENANT, + sys_metadata: ['id', 'name', 'type', 'scope', 'metadata', 'checksum'], + sys_metadata_history: ['id', 'name', 'type', 'metadata', 'checksum', 'previous_checksum'], + file_blob: ['id', 'name', 'checksum'], + }; + const view = (label: string) => JSON.stringify({ name: 'v', type: 'grid', label }); + const hashFree = (text: string) => { + expect(text).not.toContain(HASH); + expect(text).not.toContain(PARENT); + expect(text).not.toContain(NEXT); + expect(text).not.toMatch(/\\?"(previous_)?checksum\\?"/); + }; + + it('a sys_metadata create: neither the ledger snapshot nor the activity copy carries the hash', async () => { + const { engine, fire, created } = makeEngine(SCHEMAS); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterInsert', { + object: 'sys_metadata', + input: { id: 'meta-1' }, + result: { id: 'meta-1', name: 'v', type: 'view', scope: 'platform', metadata: view('one'), checksum: HASH }, + session: { userId: 'admin-1' }, + }); + const audit = created.find((c) => c.object === 'sys_audit_log'); + const activity = created.find((c) => c.object === 'sys_activity'); + hashFree(JSON.stringify(audit!.row)); + hashFree(JSON.stringify(activity!.row)); + // The copy still records the change: the other columns survive. + const newValue = JSON.parse(audit!.row.new_value); + expect(newValue.type).toBe('view'); + expect(newValue.metadata).toBe(view('one')); + }); + + it('a sys_metadata update: the diff carries the body change and not the hash change', async () => { + const { engine, fire, created } = makeEngine(SCHEMAS); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterUpdate', { + object: 'sys_metadata', + input: { id: 'meta-1', data: { metadata: view('two') } }, + previous: { id: 'meta-1', name: 'v', type: 'view', metadata: view('one'), checksum: HASH }, + result: { id: 'meta-1', name: 'v', type: 'view', metadata: view('two'), checksum: NEXT }, + session: { userId: 'admin-1' }, + }); + const audit = created.find((c) => c.object === 'sys_audit_log'); + expect(audit).toBeDefined(); + hashFree(JSON.stringify(audit!.row)); + expect(JSON.parse(audit!.row.new_value).metadata).toBe(view('two')); + hashFree(JSON.stringify(created.find((c) => c.object === 'sys_activity')!.row)); + }); + + it('a sys_metadata_history append: neither hash column is copied', async () => { + const { engine, fire, created } = makeEngine(SCHEMAS); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterInsert', { + object: 'sys_metadata_history', + input: { id: 'h-1' }, + result: { id: 'h-1', name: 'v', type: 'view', metadata: view('one'), checksum: NEXT, previous_checksum: PARENT }, + session: { userId: 'admin-1' }, + }); + for (const c of created) hashFree(JSON.stringify(c.row)); + }); + + it('a history append whose change note quotes a hash: the copy keeps the note and withholds the quote', async () => { + const { engine, fire, created } = makeEngine({ ...SCHEMAS, sys_metadata_history: [...SCHEMAS.sys_metadata_history, 'change_note'] }); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterInsert', { + object: 'sys_metadata_history', + input: { id: 'h-2' }, + result: { id: 'h-2', name: 'v', type: 'view', metadata: view('one'), checksum: NEXT, change_note: `publish draft (hash ${NEXT})` }, + session: { userId: 'admin-1' }, + }); + for (const c of created) hashFree(JSON.stringify(c.row)); + const audit = created.find((c) => c.object === 'sys_audit_log'); + expect(JSON.parse(audit!.row.new_value).change_note).toBe('publish draft (hash (withheld))'); + }); + + it('a decision-audit note that quotes a hash (its rewrite, or a row written before) is copied withheld', async () => { + const { engine, fire, created } = makeEngine({ ...SCHEMAS, sys_metadata_audit: ['id', 'code', 'note'] }); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterUpdate', { + object: 'sys_metadata_audit', + input: { id: 'd-1', data: { note: 'expected parent (withheld) but current is (withheld)' } }, + // adr0112-ok: D6b persisted audit column + previous: { id: 'd-1', code: 'metadata_conflict', note: `expected parent ${PARENT} but current is ${HASH}` }, + // adr0112-ok: D6b persisted audit column + result: { id: 'd-1', code: 'metadata_conflict', note: 'expected parent (withheld) but current is (withheld)' }, + session: {}, + }); + expect(created.length).toBeGreaterThan(0); + for (const c of created) hashFree(JSON.stringify(c.row)); + }); + + it('control: another object keeps its own checksum column in the copy', async () => { + const { engine, fire, created } = makeEngine(SCHEMAS); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterInsert', { + object: 'file_blob', + input: { id: 'f-1' }, + result: { id: 'f-1', name: 'blob', checksum: HASH }, + session: { userId: 'admin-1' }, + }); + const audit = created.find((c) => c.object === 'sys_audit_log'); + expect(JSON.parse(audit!.row.new_value).checksum).toBe(HASH); + }); +}); diff --git a/packages/plugins/plugin-audit/src/audit-writers.ts b/packages/plugins/plugin-audit/src/audit-writers.ts index 437819f06fb..dbb8419561e 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.ts @@ -43,6 +43,14 @@ import { STORED_METADATA_BODY_COLUMN, STORED_METADATA_TYPE_COLUMN, } from '@objectstack/spec/kernel'; +// [#21207] The same rows' stored content-hash columns — defined beside the +// at-rest rewrite that withholds the copies already written. +import { + METADATA_DECISION_AUDIT_OBJECT, + STORED_METADATA_HASH_COLUMNS, + STORED_METADATA_HASH_NOTE_COLUMN, + withheldStoredHashTokens, +} from './stored-metadata-body-migration.js'; // [commit 1408fe385 / #10101] The platform-row organization resolver, imported rather // than owned. It started life in THIS file (commit 1408fe385, honouring #8287's ruling) // and was promoted to `@objectstack/metadata-core` by the maintainer ruling @@ -1422,6 +1430,24 @@ export function installAuditWriters( if (outcome.ok) out[STORED_METADATA_BODY_COLUMN] = outcome.body; else delete out[STORED_METADATA_BODY_COLUMN]; } + // [#21207] …and the same row's stored CONTENT HASH is not copied at all + // (fork three, ruling A). It is a hash over the whole stored body, withheld + // credential material included, so a copy of it beside the projected body + // is an offline verifier, at rest and served to every ledger reader. A copy + // has no use for a version token, and the history table stays the lineage. + if (isStoredMetadataBodyObject(objectName)) { + for (const column of STORED_METADATA_HASH_COLUMNS) delete out[column]; + // …and a change note that quotes one keeps its words, not the hash. + const note = out[STORED_METADATA_HASH_NOTE_COLUMN]; + if (typeof note === 'string') out[STORED_METADATA_HASH_NOTE_COLUMN] = withheldStoredHashTokens(note); + } + // The same for a decision-audit row's note: a conflict note written before + // the protocol withheld its hashes — and the rewrite of one by + // `os migrate audit-metadata-bodies`, whose own ledger copy would otherwise + // carry the old note's hashes straight back into the ledger. + if (objectName === METADATA_DECISION_AUDIT_OBJECT && typeof out.note === 'string') { + out.note = withheldStoredHashTokens(out.note); + } if (dropComputed) { const defs = getFieldDefs(objectName); for (const key of Object.keys(out)) { diff --git a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts index c5350524e86..386a02f5484 100644 --- a/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts +++ b/packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts @@ -40,6 +40,23 @@ * left to withhold — so re-running and reading a clean report is the check, the * same posture `os migrate summary-nulls` takes. No `sys_migration` flag is * recorded: nothing gates irreversible behaviour on this rewrite. + * + * ## [#21207] The stored content hash of the same rows + * + * The same copies also carried the copied row's stored CONTENT HASH + * (`checksum`, and the history row's `previous_checksum`) — a hash over the + * whole stored body, withheld credential material included, i.e. an offline + * verifier for a guess at that material — and the protocol wrote both hashes + * of a refused optimistic-lock write into the decision-audit note + * (`sys_metadata_audit.note`, `code = 'metadata_conflict'`), which the writer + * then copied into the ledger and the activity feed again. The writers no + * longer do (fork three of the maintainer's ruling on #21207: a copy never + * carries the hash; the history table stays the lineage), and this rewrite now + * also drops the two columns from the copies already written + * ({@link withoutStoredHashColumns}) and withholds the hashes in the notes and + * their copies ({@link planDecisionNotePatch}) — the one operator-run rewrite + * this family already has, extended to a derived column of the same rows, never + * a second rewrite path. */ import { @@ -53,6 +70,169 @@ import type { IDataEngine } from '@objectstack/spec/contracts'; /** The two audit-family tables this writer copies stored metadata bodies into. */ export const STORED_METADATA_BODY_AUDIT_OBJECTS = ['sys_audit_log', 'sys_activity'] as const; +/** + * [#21207] The stored CONTENT-HASH columns of a stored-metadata-body row: + * `checksum` (both tables) and the history table's `previous_checksum`. Never + * copied by the audit writer (`audit-writers.ts`), dropped from the copies + * already written by this rewrite. The same list as + * `@objectstack/metadata-protocol`'s `STORED_METADATA_HASH_COLUMNS`, which this + * plugin cannot import; `stored-metadata-body-family.pin.test.ts` pins it to the + * object definitions. + */ +export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); + +/** + * [#21207] The history table's change note, which can QUOTE a stored content + * hash (`publish draft (hash …)` on rows written before the publish door stated + * its own message). A copy keeps the note and withholds each quote. + */ +export const STORED_METADATA_HASH_NOTE_COLUMN = 'change_note'; + +/** An unkeyed content hash quoted in free text (the keyed form's `hmac-sha256:` prefix is not one). */ +const QUOTED_STORED_HASH = /(?; + const note = record[STORED_METADATA_HASH_NOTE_COLUMN]; + const withheldNote = typeof note === 'string' ? withheldStoredHashTokens(note) : note; + const hasHash = STORED_METADATA_HASH_COLUMNS.some((column) => column in record); + if (!hasHash && withheldNote === note) return { changed: false, value: snapshot }; + const out: Record = { ...record }; + for (const column of STORED_METADATA_HASH_COLUMNS) delete out[column]; + if (withheldNote !== note) out[STORED_METADATA_HASH_NOTE_COLUMN] = withheldNote; + return { changed: true, value: out }; +} + +/** + * The conflict note with each side's value withheld — `null` kept, anything + * else `(withheld)` — exactly the sentence the protocol writes now; `undefined` + * for a note that is not that sentence or already withholds both sides. + */ +function withheldConflictNote(note: unknown): string | undefined { + if (typeof note !== 'string') return undefined; + const match = CONFLICT_NOTE.exec(note); + if (!match) return undefined; + const side = (value: string) => (value === 'null' ? 'null' : '(withheld)'); + const rewritten = `expected parent ${side(match[1] as string)} but current is ${side(match[2] as string)}`; + return rewritten === note ? undefined : rewritten; +} + +/** A decision-audit row (or a ledger snapshot of one) that is a conflict refusal: its code says so, or it carries none. */ +function isConflictDecision(record: Record): boolean { + return record.code === undefined || record.code === CONFLICT_NOTE_CODE; +} + +/** Withhold the hashes in the note of one parsed snapshot of a decision-audit row. */ +function withheldNoteSnapshot(snapshot: unknown): { changed: boolean; value: unknown } { + if (!snapshot || typeof snapshot !== 'object' || Array.isArray(snapshot)) return { changed: false, value: snapshot }; + const record = snapshot as Record; + if (!isConflictDecision(record)) return { changed: false, value: snapshot }; + const note = withheldConflictNote(record.note); + return note === undefined ? { changed: false, value: snapshot } : { changed: true, value: { ...record, note } }; +} + +/** Parse → apply `step` → re-serialize one stored JSON string column. A non-string / unparseable value is left as-is. */ +function rewriteSerialized( + serialized: unknown, + step: (parsed: unknown) => { changed: boolean; value: unknown }, +): { changed: boolean; value: unknown } { + if (typeof serialized !== 'string' || serialized === '') return { changed: false, value: serialized }; + let parsed: unknown; + try { + parsed = JSON.parse(serialized); + } catch { + return { changed: false, value: serialized }; + } + const { changed, value } = step(parsed); + return changed ? { changed: true, value: JSON.stringify(value) } : { changed: false, value: serialized }; +} + +/** + * [#21207] The patch that withholds the stored hashes a conflict's + * decision-audit note named, for one row of `table`, or `null` when there is + * nothing to withhold: + * + * - `sys_metadata_audit` — the note itself (`code = 'metadata_conflict'`); + * - `sys_audit_log` / `sys_activity` — the ledger snapshot / activity pair the + * writer copied a `sys_metadata_audit` row into. + * + * Each value becomes `(withheld)` and a `null` side stays `null` — the sentence + * the protocol writes since this card, so a rewritten note and a new one read + * the same. Idempotent. + */ +export function planDecisionNotePatch( + table: string, + row: Record, +): Record | null { + if (table === METADATA_DECISION_AUDIT_OBJECT) { + if (!isConflictDecision(row)) return null; + const note = withheldConflictNote(row.note); + return note === undefined ? null : { note }; + } + if (row.object_name !== METADATA_DECISION_AUDIT_OBJECT) return null; + if (table === 'sys_audit_log') { + const patch: Record = {}; + for (const column of ['new_value', 'old_value']) { + const out = rewriteSerialized(row[column], withheldNoteSnapshot); + if (out.changed) patch[column] = out.value; + } + return Object.keys(patch).length > 0 ? patch : null; + } + if (table === 'sys_activity') { + const out = rewriteSerialized(row.metadata, (parsed) => { + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return { changed: false, value: parsed }; + const pair = parsed as Record; + const oldR = withheldNoteSnapshot(pair.old); + const newR = withheldNoteSnapshot(pair.new); + if (!oldR.changed && !newR.changed) return { changed: false, value: parsed }; + return { changed: true, value: { ...pair, old: oldR.value, new: newR.value } }; + }); + return out.changed ? { metadata: out.value } : null; + } + return null; +} + +/** Whether a copy row is about a stored-metadata-body row — its `object_name` says so, or it names none. */ +function copiesStoredMetadataRow(row: Record): boolean { + return row.object_name === undefined || row.object_name === null + || (STORED_METADATA_BODY_OBJECTS as ReadonlySet).has(String(row.object_name)); +} + +/** Redact the body AND drop the content-hash columns of one parsed snapshot of a stored-metadata row. */ +function redactStoredMetadataSnapshot( + snapshot: unknown, + typeHint: string | undefined, + dropHashes: boolean, +): { changed: boolean; value: unknown } { + const body = redactLedgerSnapshotBody(snapshot, typeHint); + if (!dropHashes) return body; + const hashes = withoutStoredHashColumns(body.value); + return { changed: body.changed || hashes.changed, value: hashes.value }; +} + /** The rewrite reads and writes as the platform, never as a user. */ const SYSTEM_CTX = { isSystem: true } as const; @@ -87,43 +267,42 @@ export function redactLedgerSnapshotBody( return { changed: true, value: rest }; } -/** Parse → redact → re-serialize one stored JSON string column. A non-string / unparseable value is left as-is. */ +/** + * Parse → redact (and, for a stored-metadata row's copy, drop the content-hash + * columns) → re-serialize one stored JSON string column. A non-string / + * unparseable value is left as-is. + */ function redactSerializedSnapshot( serialized: unknown, - typeHint?: string, + typeHint: string | undefined, + dropHashes: boolean, ): { changed: boolean; value: unknown } { - if (typeof serialized !== 'string' || serialized === '') return { changed: false, value: serialized }; - let parsed: unknown; - try { - parsed = JSON.parse(serialized); - } catch { - return { changed: false, value: serialized }; - } - const { changed, value } = redactLedgerSnapshotBody(parsed, typeHint); - if (!changed) return { changed: false, value: serialized }; - return { changed: true, value: JSON.stringify(value) }; + return rewriteSerialized(serialized, (parsed) => redactStoredMetadataSnapshot(parsed, typeHint, dropHashes)); } /** * The patch for one `sys_audit_log` row about a stored-metadata-body object, or - * `null` when the row holds no credential body to rewrite. + * `null` when the row holds no credential body to rewrite — and [#21207] no + * stored content hash to drop. */ export function planAuditRowPatch( row: Record, typeHint?: string, ): { new_value?: unknown; old_value?: unknown } | null { + const dropHashes = copiesStoredMetadataRow(row); const patch: { new_value?: unknown; old_value?: unknown } = {}; - const nv = redactSerializedSnapshot(row.new_value, typeHint); + const nv = redactSerializedSnapshot(row.new_value, typeHint, dropHashes); if (nv.changed) patch.new_value = nv.value; - const ov = redactSerializedSnapshot(row.old_value, typeHint); + const ov = redactSerializedSnapshot(row.old_value, typeHint, dropHashes); if (ov.changed) patch.old_value = ov.value; return nv.changed || ov.changed ? patch : null; } /** * The patch for one `sys_activity` row about a stored-metadata-body object, or - * `null` when it holds no credential body. `sys_activity.metadata` is the - * serialized pair `{ old, new }`, each half a ledger snapshot. + * `null` when it holds no credential body — and [#21207] no stored content hash + * to drop. `sys_activity.metadata` is the serialized pair `{ old, new }`, each + * half a ledger snapshot. */ export function planActivityRowPatch( row: Record, @@ -139,8 +318,9 @@ export function planActivityRowPatch( } if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; const pair = parsed as Record; - const oldR = redactLedgerSnapshotBody(pair.old, typeHint); - const newR = redactLedgerSnapshotBody(pair.new, typeHint); + const dropHashes = copiesStoredMetadataRow(row); + const oldR = redactStoredMetadataSnapshot(pair.old, typeHint, dropHashes); + const newR = redactStoredMetadataSnapshot(pair.new, typeHint, dropHashes); if (!oldR.changed && !newR.changed) return null; return { metadata: JSON.stringify({ ...pair, old: oldR.value, new: newR.value }) }; } @@ -182,8 +362,10 @@ function asArray(result: unknown): Record[] { /** * Rewrite every `sys_audit_log` / `sys_activity` row that copied a stored - * metadata body in cleartext, projecting the body through the shared redactor. - * Dry-run by default; `apply` writes. Idempotent. + * metadata body in cleartext, projecting the body through the shared redactor + * — and [#21207] every copy that carries a stored content hash (dropped) and + * every conflict note in `sys_metadata_audit` and its copies that names one + * (withheld). Dry-run by default; `apply` writes. Idempotent. */ export async function migrateStoredMetadataBodyCopies( engine: StoredMetadataBodyMigrationEngine, @@ -218,15 +400,21 @@ export async function migrateStoredMetadataBodyCopies( return type; }; - for (const object of STORED_METADATA_BODY_AUDIT_OBJECTS) { + // [#21207] The copies of a decision-audit row are read too (its conflict + // note named both stored hashes), and so is the decision-audit table itself. + const copiedObjects = [...STORED_METADATA_BODY_OBJECTS, METADATA_DECISION_AUDIT_OBJECT]; + const tables: ReadonlyArray<{ object: string; where: Record }> = [ + ...STORED_METADATA_BODY_AUDIT_OBJECTS.map((object) => ({ object, where: { object_name: { $in: copiedObjects } } })), + // A PREDICATE on the persisted `code` column (ADR-0112 D6b, the audit + // column's own vocabulary), written in operator form: it reads rows by the + // code they carry and stamps none. + { object: METADATA_DECISION_AUDIT_OBJECT, where: { code: { $eq: CONFLICT_NOTE_CODE } } }, + ]; + for (const { object, where } of tables) { report.byObject[object] = { scanned: 0, rewritten: 0 }; let rows: Record[]; try { - const result = await engine.find( - object, - { where: { object_name: { $in: [...STORED_METADATA_BODY_OBJECTS] } } }, - { context: SYSTEM_CTX }, - ); + const result = await engine.find(object, { where }, { context: SYSTEM_CTX }); rows = asArray(result); } catch (e) { // A table this run could not read is NOT a clean table: counted as a @@ -243,9 +431,13 @@ export async function migrateStoredMetadataBodyCopies( for (const row of rows) { report.scanned += 1; report.byObject[object].scanned += 1; - const typeHint = await resolveType(row.record_id); - const patch = - object === 'sys_audit_log' ? planAuditRowPatch(row, typeHint) : planActivityRowPatch(row, typeHint); + // [#21207] A decision-audit row, or a copy of one: its conflict note. + const isDecision = object === METADATA_DECISION_AUDIT_OBJECT || row.object_name === METADATA_DECISION_AUDIT_OBJECT; + const patch = isDecision + ? planDecisionNotePatch(object, row) + : object === 'sys_audit_log' + ? planAuditRowPatch(row, await resolveType(row.record_id)) + : planActivityRowPatch(row, await resolveType(row.record_id)); if (!patch) continue; report.rewritten += 1; report.byObject[object].rewritten += 1; @@ -272,7 +464,7 @@ export async function migrateStoredMetadataBodyCopies( logger.info?.( `[stored-metadata-body-migration] ${apply ? 'rewrote' : 'would rewrite'} ${report.rewritten} of ${report.scanned} ` + - `audit/activity row(s) carrying a stored metadata body` + + `audit/activity/decision row(s) carrying a stored metadata body or a stored content hash` + (report.failures > 0 ? ` (${report.failures} failed — re-run to finish)` : ''), ); return report; diff --git a/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts new file mode 100644 index 00000000000..e5a3050ea9b --- /dev/null +++ b/packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts @@ -0,0 +1,191 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21207] Exit two, fork three, at rest: the copies already written carry no + * stored content hash after `os migrate audit-metadata-bodies` runs. + * + * Before this card the audit writer copied a `sys_metadata` / + * `sys_metadata_history` row's `checksum` (and the history row's + * `previous_checksum`) into the ledger snapshot / diff and the activity copy, + * and the protocol wrote both hashes of a refused optimistic-lock write into + * the decision-audit note (`sys_metadata_audit.note`) — which the writer then + * copied into the ledger again. The writers no longer do; the same operator-run + * migration that withholds the copied body now also drops the two columns from + * those copies and withholds the hashes in those notes. Dry run by default, + * idempotent, and the history table stays the lineage. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { assertEngineFindOnePredicate, assertEngineUpdateDispatch } from '@objectstack/metadata-core'; +import { + migrateStoredMetadataBodyCopies, + planActivityRowPatch, + planAuditRowPatch, + planDecisionNotePatch, + type StoredMetadataBodyMigrationEngine, +} from './stored-metadata-body-migration.js'; + +const HASH = `sha256:${'a'.repeat(64)}`; +const PARENT = `sha256:${'b'.repeat(64)}`; +const SHA256 = /sha256:[0-9a-f]{64}/; +const HASH_KEY = /"(previous_)?checksum"/; + +const view = (label: string) => JSON.stringify({ name: 'v', type: 'grid', label }); +const metaSnapshot = () => ({ id: 'm1', name: 'v', type: 'view', scope: 'platform', metadata: view('one'), checksum: HASH }); +const historySnapshot = () => ({ id: 'h1', name: 'v', type: 'view', metadata: view('one'), checksum: HASH, previous_checksum: PARENT }); +const conflictNote = `expected parent ${PARENT} but current is ${HASH}`; +const WITHHELD_NOTE = 'expected parent (withheld) but current is (withheld)'; +/** The decision-audit `code` column's own vocabulary (ADR-0112 D6b), not an error code. */ +const CONFLICT_CODE = 'metadata_conflict'; // adr0112-ok: D6b persisted audit column + +describe('planAuditRowPatch / planActivityRowPatch — the hash columns leave the copy', () => { + it('a sys_metadata create snapshot loses its checksum, and nothing else', () => { + const row = { id: 'a1', object_name: 'sys_metadata', record_id: 'm1', new_value: JSON.stringify(metaSnapshot()), old_value: null }; + const patch = planAuditRowPatch(row); + expect(patch).not.toBeNull(); + const rewritten = JSON.parse(String(patch!.new_value)); + expect(rewritten).toEqual({ id: 'm1', name: 'v', type: 'view', scope: 'platform', metadata: view('one') }); + }); + + it('a sys_metadata_history snapshot loses both hash columns', () => { + const row = { id: 'a2', object_name: 'sys_metadata_history', record_id: 'h1', new_value: JSON.stringify(historySnapshot()), old_value: null }; + const patch = planAuditRowPatch(row); + expect(String(patch!.new_value)).not.toMatch(SHA256); + expect(String(patch!.new_value)).not.toMatch(HASH_KEY); + }); + + it('an update diff loses the hash on both sides', () => { + const row = { + id: 'a3', + object_name: 'sys_metadata', + record_id: 'm1', + old_value: JSON.stringify({ metadata: view('one'), checksum: PARENT }), + new_value: JSON.stringify({ metadata: view('two'), checksum: HASH }), + }; + const patch = planAuditRowPatch(row, 'view'); + expect(JSON.stringify(patch)).not.toMatch(SHA256); + expect(JSON.parse(String(patch!.new_value)).metadata).toBe(view('two')); + }); + + it('the activity pair loses the hash on both halves', () => { + const row = { id: 'ac1', object_name: 'sys_metadata_history', record_id: 'h1', metadata: JSON.stringify({ old: null, new: historySnapshot() }) }; + const patch = planActivityRowPatch(row); + expect(patch!.metadata).not.toMatch(SHA256); + expect(JSON.parse(patch!.metadata).new.name).toBe('v'); + }); + + it('is idempotent, and leaves another object\'s checksum column alone', () => { + const row = { id: 'a1', object_name: 'sys_metadata', record_id: 'm1', new_value: JSON.stringify(metaSnapshot()), old_value: null }; + const once = planAuditRowPatch(row)!; + expect(planAuditRowPatch({ ...row, new_value: once.new_value })).toBeNull(); + const foreign = { id: 'a9', object_name: 'file_blob', record_id: 'f1', new_value: JSON.stringify({ id: 'f1', checksum: HASH }), old_value: null }; + expect(planAuditRowPatch(foreign)).toBeNull(); + }); +}); + +describe('planDecisionNotePatch — the decision-audit note and its copies', () => { + it('withholds both hashes of a conflict note', () => { + const patch = planDecisionNotePatch('sys_metadata_audit', { id: 'd1', code: CONFLICT_CODE, note: conflictNote }); + expect(patch).toEqual({ note: WITHHELD_NOTE }); + }); + + it('keeps a null side as null', () => { + const patch = planDecisionNotePatch('sys_metadata_audit', { id: 'd2', code: CONFLICT_CODE, note: `expected parent null but current is ${HASH}` }); + expect(patch).toEqual({ note: 'expected parent null but current is (withheld)' }); + }); + + it('leaves every other note alone, and is idempotent', () => { + expect(planDecisionNotePatch('sys_metadata_audit', { id: 'd3', code: 'ok', note: 'restored from version 2' })).toBeNull(); + expect(planDecisionNotePatch('sys_metadata_audit', { id: 'd4', code: CONFLICT_CODE, note: WITHHELD_NOTE })).toBeNull(); + }); + + it('rewrites the ledger and activity copies of a conflict note', () => { + const snapshot = { id: 'd1', type: 'view', name: 'v', operation: 'save', outcome: 'denied', code: CONFLICT_CODE, note: conflictNote }; + const audit = planDecisionNotePatch('sys_audit_log', { + id: 'a5', object_name: 'sys_metadata_audit', record_id: 'd1', new_value: JSON.stringify(snapshot), old_value: null, + }); + expect(String(audit!.new_value)).not.toMatch(SHA256); + expect(JSON.parse(String(audit!.new_value)).note).toBe(WITHHELD_NOTE); + const activity = planDecisionNotePatch('sys_activity', { + id: 'ac5', object_name: 'sys_metadata_audit', record_id: 'd1', metadata: JSON.stringify({ old: null, new: snapshot }), + }); + expect(String(activity!.metadata)).not.toMatch(SHA256); + }); +}); + +describe('migrateStoredMetadataBodyCopies — the content-hash copies (driven)', () => { + function fakeEngine(rows: Record[]>) { + const updates: Array<{ object: string; id: string; data: Record }> = []; + const engine: StoredMetadataBodyMigrationEngine = { + async find(object) { + return rows[object] ?? []; + }, + async findOne(object, query) { + assertEngineFindOnePredicate(object, query); + const id = (query as { where?: { id?: unknown } } | undefined)?.where?.id; + return (rows[object] ?? []).find((r) => r.id === id) ?? null; + }, + async update(object, data, options) { + assertEngineUpdateDispatch(data, options === undefined ? undefined : { ...options }); + const { id, ...rest } = data as Record; + updates.push({ object, id: String(id), data: rest }); + const target = (rows[object] ?? []).find((r) => r.id === id); + if (target) Object.assign(target, rest); + return { id, ...rest }; + }, + }; + return { engine, updates }; + } + const logger = { info: vi.fn(), warn: vi.fn() }; + const fixture = () => ({ + sys_metadata: [{ id: 'm1', type: 'view', name: 'v' }], + sys_audit_log: [ + { id: 'a1', object_name: 'sys_metadata', record_id: 'm1', new_value: JSON.stringify(metaSnapshot()), old_value: null }, + { id: 'a2', object_name: 'sys_metadata_audit', record_id: 'd1', new_value: JSON.stringify({ id: 'd1', code: CONFLICT_CODE, note: conflictNote }), old_value: null }, + { id: 'a3', object_name: 'file_blob', record_id: 'f1', new_value: JSON.stringify({ id: 'f1', checksum: HASH }), old_value: null }, + ], + sys_activity: [{ id: 'ac1', object_name: 'sys_metadata_history', record_id: 'h1', metadata: JSON.stringify({ old: null, new: historySnapshot() }) }], + sys_metadata_audit: [ + { id: 'd1', code: CONFLICT_CODE, note: conflictNote }, + { id: 'd2', code: 'ok', note: 'active' }, + ], + }); + + it('dry run counts every copy carrying a hash, per table, and writes nothing', async () => { + const { engine, updates } = fakeEngine(fixture()); + const report = await migrateStoredMetadataBodyCopies(engine, logger, { apply: false }); + expect(updates).toHaveLength(0); + expect(report.byObject.sys_audit_log!.rewritten).toBe(2); + expect(report.byObject.sys_activity!.rewritten).toBe(1); + expect(report.byObject.sys_metadata_audit!.rewritten).toBe(1); + expect(report.rewritten).toBe(4); + }); + + it('--apply rewrites them, the written values carry no hash, and a second run finds nothing', async () => { + const rows = fixture(); + const { engine, updates } = fakeEngine(rows); + const first = await migrateStoredMetadataBodyCopies(engine, logger, { apply: true }); + expect(first.failures).toBe(0); + expect(updates.map((u) => `${u.object}/${u.id}`).sort()).toEqual([ + 'sys_activity/ac1', 'sys_audit_log/a1', 'sys_audit_log/a2', 'sys_metadata_audit/d1', + ]); + for (const u of updates) expect(JSON.stringify(u.data)).not.toMatch(SHA256); + // Another object's checksum column was not touched. + expect(String(rows.sys_audit_log[2]!.new_value)).toContain(HASH); + + const second = await migrateStoredMetadataBodyCopies(engine, logger, { apply: false }); + expect(second.rewritten).toBe(0); + }); +}); + +describe('the history change note that quotes a stored hash (#21207)', () => { + it('a copied history snapshot keeps its note and withholds the quote, idempotently', () => { + const snapshot = { ...historySnapshot(), change_note: `publish draft (hash ${HASH})` }; + const row = { id: 'a7', object_name: 'sys_metadata_history', record_id: 'h1', new_value: JSON.stringify(snapshot), old_value: null }; + const patch = planAuditRowPatch(row)!; + const rewritten = JSON.parse(String(patch.new_value)); + expect(rewritten.change_note).toBe('publish draft (hash (withheld))'); + expect(String(patch.new_value)).not.toMatch(SHA256); + expect(planAuditRowPatch({ ...row, new_value: patch.new_value })).toBeNull(); + }); +}); diff --git a/packages/rest/src/rest-server.ts b/packages/rest/src/rest-server.ts index 765ce8c9990..53ed10a9399 100644 --- a/packages/rest/src/rest-server.ts +++ b/packages/rest/src/rest-server.ts @@ -6981,10 +6981,13 @@ export class RestServer { : body; // Opt-in OCC under ADR-0008 PR-10d.3: callers (Studio, - // CLI) may set `If-Match: ` to enforce that - // the overlay row has not advanced since they last read - // it. A `null`/empty body or no header preserves the - // legacy last-write-wins behaviour. + // CLI) may set `If-Match` to the version token a receipt + // served, to enforce that the overlay row has not advanced + // since they last read it. A `null`/empty body or no header + // preserves the legacy last-write-wins behaviour. [#21207] + // The token is the crypto provider's keyed digest of the + // stored content hash, never the hash itself; the protocol + // compares it in that form, so it passes through here as sent. const ifMatchHeader = req.headers?.['if-match'] ?? req.headers?.['If-Match']; const parentVersion = typeof ifMatchHeader === 'string' ? ifMatchHeader.replace(/^"|"$/g, '') // strip ETag-style quotes diff --git a/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts b/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts index ef9030b4475..033306806ef 100644 --- a/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts +++ b/packages/services/service-analytics/src/stored-metadata-body-refusal.test.ts @@ -70,3 +70,55 @@ describe('storedMetadataBodyAnalyticsRefusal', () => { expect(err?.field).toBe('metadata'); }); }); + +/** + * [#21207] The two stored content-hash columns of the same tables (`checksum`, + * and the history table's `previous_checksum`). Each is a hash over the WHOLE + * stored body, withheld credential material included, so a dimension serves an + * offline verifier and a filter is an online one. They are refused in the same + * envelope as the body column, in either role, on both tables. + */ +describe('storedMetadataBodyAnalyticsRefusal — the content-hash columns (#21207)', () => { + const HASH_COLUMNS: Array<[string, string]> = [ + ['sys_metadata', 'checksum'], + ['sys_metadata_history', 'checksum'], + ['sys_metadata_history', 'previous_checksum'], + ]; + for (const [object, column] of HASH_COLUMNS) { + it(`refuses '${column}' on ${object} as a dimension or measure (INVALID_FIELD / 400, param dimensions)`, () => { + const err = storedMetadataBodyAnalyticsRefusal([field(object, column, 'aggregate')]) as any; + expect(err).toBeInstanceOf(Error); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe(column); + expect(err.object).toBe(object); + expect(err.param).toBe('dimensions'); + // The refusal names the usable columns. + expect(err.message).toContain("'type'"); + }); + + it(`refuses '${column}' on ${object} as a filter or sort (param where)`, () => { + const err = storedMetadataBodyAnalyticsRefusal([field(object, column, 'predicate')]) as any; + expect(err?.code).toBe('INVALID_FIELD'); + expect(err?.status).toBe(400); + expect(err?.field).toBe(column); + expect(err?.param).toBe('where'); + }); + } + + it('leaves a `checksum` column on an object outside the family alone', () => { + expect(storedMetadataBodyAnalyticsRefusal([field('file_blob', 'checksum', 'aggregate')])).toBeUndefined(); + expect(storedMetadataBodyAnalyticsRefusal([field('file_blob', 'previous_checksum', 'predicate')])).toBeUndefined(); + }); +}); + +describe('storedMetadataBodyAnalyticsRefusal — the history change note (#21207)', () => { + it('refuses the change note, which can quote a stored hash, in either role', () => { + for (const role of ['aggregate', 'predicate'] as const) { + const err = storedMetadataBodyAnalyticsRefusal([field('sys_metadata_history', 'change_note', role)]) as any; + expect(err?.code).toBe('INVALID_FIELD'); + expect(err?.status).toBe(400); + expect(err?.field).toBe('change_note'); + } + }); +}); diff --git a/packages/services/service-analytics/src/stored-metadata-body-refusal.ts b/packages/services/service-analytics/src/stored-metadata-body-refusal.ts index 1b27826be3e..9be677a07df 100644 --- a/packages/services/service-analytics/src/stored-metadata-body-refusal.ts +++ b/packages/services/service-analytics/src/stored-metadata-body-refusal.ts @@ -34,10 +34,34 @@ * Every OTHER member of these objects — `type`, `name`, `scope`, `state`, * timestamps — is grouped, filtered and counted as before, so the Setup grids * and "All Metadata" dashboards that chart metadata by type keep working. Only - * the body column is refused. + * the body column, and since #21207 the two stored content-hash columns + * ({@link STORED_METADATA_HASH_COLUMNS}), are refused. */ import { isStoredMetadataBodyObject, STORED_METADATA_BODY_COLUMN } from '@objectstack/spec/kernel'; + +/** + * [#21207] The stored CONTENT-HASH columns of the same two tables: `checksum` + * (both) and the history table's `previous_checksum` (the parent's hash). + * + * Each is the canonical SHA-256 of the WHOLE stored body, withheld credential + * material included. Grouped by, it serves the stored values — beside the + * projected body every other door serves, an offline verifier for a guess at + * the withheld material; filtered on, it is an online one. Per the + * maintainer's ruling on #21207 they are refused here in the body column's + * envelope, in either role. The same list as + * `@objectstack/metadata-protocol`'s `STORED_METADATA_HASH_COLUMNS`, which this + * service cannot import; `stored-metadata-body-family.pin.test.ts` pins it to + * the object definitions. + */ +export const STORED_METADATA_HASH_COLUMNS: readonly string[] = Object.freeze(['checksum', 'previous_checksum']); + +/** + * [#21207] The history table's change note, which can QUOTE a stored content + * hash (`publish draft (hash …)` on rows written before the publish door stated + * its own message) — refused as a member for the same reason. + */ +export const STORED_METADATA_HASH_NOTE_COLUMN = 'change_note'; import type { StandardErrorCode } from '@objectstack/spec/api'; import type { NamedField, NamedRead } from './field-read-admission.js'; @@ -57,18 +81,28 @@ export function storedMetadataBodyAnalyticsRefusal( // it outright (`field-read-admission.ts`, the expression refusal) — one // rule for expressions, never a second one in this module. if ('expression' in f) continue; - if (!isStoredMetadataBodyObject(f.object) || f.field !== STORED_METADATA_BODY_COLUMN) continue; + if (!isStoredMetadataBodyObject(f.object)) continue; + const isBody = f.field === STORED_METADATA_BODY_COLUMN; + if (!isBody && !STORED_METADATA_HASH_COLUMNS.includes(f.field) && f.field !== STORED_METADATA_HASH_NOTE_COLUMN) continue; const param = f.role === 'aggregate' ? 'dimensions' : 'where'; const err = new Error( - `Cannot query '${f.object}' by '${STORED_METADATA_BODY_COLUMN}': the query was not run. The ` - + `${STORED_METADATA_BODY_COLUMN} column holds a stored metadata body, with stored credential material ` - + `withheld on every read exit; grouping, aggregating, filtering or sorting by it would evaluate the ` - + `stored body (a group key that cannot be projected, or a filter oracle that rebuilds a withheld value ` - + `by probing). Group, filter or sort by 'type', 'name' or another scalar column instead.`, + isBody + ? `Cannot query '${f.object}' by '${STORED_METADATA_BODY_COLUMN}': the query was not run. The ` + + `${STORED_METADATA_BODY_COLUMN} column holds a stored metadata body, with stored credential material ` + + `withheld on every read exit; grouping, aggregating, filtering or sorting by it would evaluate the ` + + `stored body (a group key that cannot be projected, or a filter oracle that rebuilds a withheld value ` + + `by probing). Group, filter or sort by 'type', 'name' or another scalar column instead.` + // [#21207] A stored content-hash column, in the same envelope. + : `Cannot query '${f.object}' by '${f.field}': the query was not run. The ${f.field} column ` + + `${f.field === STORED_METADATA_HASH_NOTE_COLUMN ? 'can quote' : 'holds'} the ` + + `stored content hash of a metadata body, computed over withheld credential material too, so every read ` + + `exit serves it only in keyed form; grouping, aggregating, filtering or sorting by it would evaluate the ` + + `stored hash (a group key that serves it, or a filter that confirms a guessed hash). Group, filter or ` + + `sort by 'type', 'name' or another scalar column instead.`, ) as Error & { code: string; status: number; field: string; object: string; param: string }; err.code = INVALID_FIELD; err.status = 400; - err.field = STORED_METADATA_BODY_COLUMN; + err.field = f.field; err.object = f.object; err.param = param; return err; diff --git a/scripts/engine-double-contract.pinned.json b/scripts/engine-double-contract.pinned.json index 5f90bd3b1f8..ebc6f6b26cc 100644 --- a/scripts/engine-double-contract.pinned.json +++ b/scripts/engine-double-contract.pinned.json @@ -486,6 +486,11 @@ "verb": "update", "pinned": 1 }, + { + "file": "packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts", + "verb": "findOne", + "pinned": 2 + }, { "file": "packages/metadata-protocol/src/protocol.data-door-stored-metadata-redaction.test.ts", "verb": "findOne", @@ -1331,6 +1336,21 @@ "verb": "update", "pinned": 1 }, + { + "file": "packages/metadata-protocol/src/protocol.served-content-hash.test.ts", + "verb": "delete", + "pinned": 1 + }, + { + "file": "packages/metadata-protocol/src/protocol.served-content-hash.test.ts", + "verb": "findOne", + "pinned": 1 + }, + { + "file": "packages/metadata-protocol/src/protocol.served-content-hash.test.ts", + "verb": "update", + "pinned": 1 + }, { "file": "packages/metadata-protocol/src/protocol.stored-conversions.test.ts", "verb": "delete", @@ -2491,6 +2511,16 @@ "verb": "update", "pinned": 1 }, + { + "file": "packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts", + "verb": "findOne", + "pinned": 1 + }, + { + "file": "packages/plugins/plugin-audit/src/stored-metadata-hash-migration.test.ts", + "verb": "update", + "pinned": 1 + }, { "file": "packages/plugins/plugin-auth/src/accept-invitation-adopt-membership.test.ts", "verb": "delete",