From 1fdca2ed718a600a46b72f2214351821a96da653 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 16:35:03 +0000 Subject: [PATCH 1/6] fix(metadata-protocol)!: the door exports its stored-metadata search narrowing and filter-field collector; the reader seam calls both (WIP) Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- packages/metadata-protocol/src/index.ts | 7 + .../src/metadata-redaction.ts | 5 +- packages/metadata-protocol/src/protocol.ts | 283 ++++++++++++++---- .../src/stored-metadata-reader-seam.ts | 125 ++------ 4 files changed, 260 insertions(+), 160 deletions(-) diff --git a/packages/metadata-protocol/src/index.ts b/packages/metadata-protocol/src/index.ts index 7b911fbd533..b938c1a5d61 100644 --- a/packages/metadata-protocol/src/index.ts +++ b/packages/metadata-protocol/src/index.ts @@ -220,6 +220,13 @@ export { storedMetadataHashEvaluateRefusal, storedMetadataSearchRefusal, } from './metadata-redaction.js'; +// [#21544] …and what those refusals are FED: the door's one default-search +// narrowing and its one filter-field collector, module functions the door +// itself calls. Exported so the reader-context seam calls the same two +// functions rather than re-stating either — a second control flow over the +// same columns is where the two doors would drift. +export { collectStoredMetadataFilterFields, narrowStoredMetadataSearch } from './protocol.js'; +export type { StoredMetadataSearchSchema } from './protocol.js'; export type { MetadataHostEngine } from './host-engine.js'; diff --git a/packages/metadata-protocol/src/metadata-redaction.ts b/packages/metadata-protocol/src/metadata-redaction.ts index b4250d48d73..9a22cc8a0ca 100644 --- a/packages/metadata-protocol/src/metadata-redaction.ts +++ b/packages/metadata-protocol/src/metadata-redaction.ts @@ -742,8 +742,9 @@ export function storedMetadataBodyGroupingRefusal(object: string, groupBy: unkno * the grouping refusal above take. Same family, shape and code: `INVALID_FIELD` * / 400, naming the field, the object and the offending `param`. * - * `filterFields` is the set of head field names the caller's `where` names - * (`collectFilterFieldKeys`), and `sortFields` the fields its `orderBy` names. + * `filterFields` is the set of columns the caller's filters read + * (`collectStoredMetadataFilterFields`, `protocol.ts`: each key's head and each + * cross-field comparand's), and `sortFields` the fields its `orderBy` names. * Filter is judged before sort — a query that does both reads "the filter was * not run" first. `undefined` when neither names the body column. */ diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 24cd0629f98..fa64d2b26cd 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -119,6 +119,7 @@ import { AggregationFunction, DateGranularity, resolveSearchFieldResolution, SEARCHABLE_TEXTUAL_TYPES, SEARCHABLE_ENUM_TYPES, SEARCH_AUTO_EXCLUDED_FIELDS, isVirtualSearchField, + type SearchFieldMeta, classifyDottedFilterHead, foldQueryAliasSlots, QUERY_TRANSPORT_ALIAS_SLOTS, QUERY_TRANSPORT_DOLLAR_ALIASES, QUERY_TRANSPORT_DOLLAR_PARAMS, @@ -209,7 +210,6 @@ import { serveStoredHashTokens, serveStoredMetadataHashColumnRows, serveStoredMetadataHashColumns, - STORED_METADATA_UNSEARCHABLE_COLUMNS, storedMetadataHashEvaluateRefusal, storedMetadataSearchRefusal, type StoredHashDigest, @@ -3962,6 +3962,13 @@ const FILTER_LOGICAL_KEYS: ReadonlySet = new Set(['$and', '$or', '$not'] * produce one, but `POST /data/:object/query` is not the only door — the RPC * dispatcher and in-process callers hand over live objects — and a gate that * can hang the read path is worse than the defect it closes. + * + * [#21544] ⛔ This answers the ingress gate's question — does each named KEY + * exist — and its failure direction (a hole, never a false 400) is right only + * for that question. It is NOT what the stored-metadata family's evaluate + * refusals read: a filter can read a column without naming it as a key (a + * cross-field comparand), and a hole there is a leak, not a missing 400. That + * question has its own collector, {@link collectStoredMetadataFilterFields}. */ function collectFilterFieldKeys( where: unknown, @@ -3985,6 +3992,200 @@ function collectFilterFieldKeys( return out; } +/** + * [#21544] The stored-metadata family's ONE filter-field collector: every column + * a read query's filters READ, judged the way the family's evaluate refusals + * ({@link storedMetadataBodyPredicateRefusal}, + * {@link storedMetadataHashEvaluateRefusal}) need it. The generic data door + * calls it, and so does the in-process reader-context seam in + * `@objectstack/runtime` (`stored-metadata-reader-seam.ts`), on the same query, + * so the two answer every filter identically. `[]` for an object outside the + * family ({@link isStoredMetadataBodyObject}) or a query that is not a record. + * + * The filter positions it reads are a read query's three: `where`, `filter` + * (the engine option alias a direct engine call may still carry) and each + * `aggregations[i].filter`. ⛔ Not `having`: its keys and references name the + * AGGREGATED row's columns — group keys and aggregation aliases — and the + * engine refuses any other name before a row exists, so a family column can + * never be read there, while an alias that happens to be spelled like one + * (`{ function: 'count', alias: 'metadata' }`) is a legitimate count. + * + * What counts as a read, and why it is more than the ingress gate's + * {@link collectFilterFieldKeys} collects — each rule a measured reach at the + * generic data door before this collector existed: + * + * - **A cross-field comparand is a read.** `{ name: { $eq: { $field: 'metadata' } } }` + * compares each row's `name` with its stored body, and the SQL drivers + * evaluate it: row presence discloses the referenced column exactly as a + * filter on it would. Every node under a field key that carries a string + * `$field` — an operator bag, a list member, an `addDays` offset — is a read + * of that column: the reference shape `driver-sql` resolves (the spec's + * `FieldReferenceSchema`), taken without its other refinements, so a + * malformed reference is refused here rather than read. + * - **Depth never hides a read.** The walk is iterative and visits every node + * once per reading, so it terminates on a self-referential live object + * without a depth backstop: the ingress collector's backstop stops at 32 + * levels, and a body predicate nested below it was evaluated unrefused. + * - **A dotted name reads its head.** `metadata.x` reads `metadata`, as a + * key and as a reference. + * - **A `$` key is never a column, and what sits beneath one is read as a + * condition** — the three declared combinators and any other alike, so an + * unrecognised combinator cannot hide a family column (the engine refuses + * one anyway; the refusal just arrives first). + * - **A field key's value is read for references only.** The keys of a + * nested-relation condition (`{ owner: { region: 'NA' } }`) are another + * object's columns; a `$field` beneath one is still collected, so the + * ambiguous spelling is refused rather than resolved. + * - **A `FilterArray` is lowered first** (`parseFilterAST`), the sugar a direct + * engine call still honours on `where`, so either authoring form reads the + * same. + * + * Each column is returned once. + */ +export function collectStoredMetadataFilterFields(object: string, query: unknown): string[] { + if (!isStoredMetadataBodyObject(object)) return []; + if (query === null || typeof query !== 'object' || Array.isArray(query)) return []; + const bag = query as Record; + const filters: unknown[] = [bag.where, bag.filter]; + if (Array.isArray(bag.aggregations)) { + for (const aggregation of bag.aggregations) { + if (aggregation !== null && typeof aggregation === 'object') { + filters.push((aggregation as { filter?: unknown }).filter); + } + } + } + const out = new Set(); + for (const filter of filters) collectFilterReads(isFilterAST(filter) ? parseFilterAST(filter) : filter, out); + return [...out]; +} + +/** The head segment of a column name: `metadata.x` reads `metadata`. */ +function headSegment(name: string): string { + return name.split('.')[0] as string; +} + +/** + * The walk {@link collectStoredMetadataFilterFields} runs over one filter + * condition. A node is visited as a CONDITION (its non-`$` keys are columns) + * or as a COMPARAND (the value under a column key, where only a `$field` + * reference is a read), at most once per reading — two readings, because a + * live object reached once as a comparand and once as a condition must be read + * both ways. + */ +function collectFilterReads(root: unknown, out: Set): void { + const seenAsCondition = new WeakSet(); + const seenAsComparand = new WeakSet(); + const pending: Array<{ node: unknown; condition: boolean }> = [{ node: root, condition: true }]; + while (pending.length > 0) { + const { node, condition } = pending.pop() as { node: unknown; condition: boolean }; + if (node === null || typeof node !== 'object') continue; + const seen = condition ? seenAsCondition : seenAsComparand; + if (seen.has(node)) continue; + seen.add(node); + if (Array.isArray(node)) { + for (const member of node) pending.push({ node: member, condition }); + continue; + } + const record = node as Record; + if (!condition && typeof record.$field === 'string') out.add(headSegment(record.$field)); + for (const [key, value] of Object.entries(record)) { + if (condition && !key.startsWith('$')) { + out.add(headSegment(key)); + pending.push({ node: value, condition: false }); + } else { + pending.push({ node: value, condition }); + } + } + } +} + +/** + * The slice of an object definition {@link narrowStoredMetadataSearch} reads: + * the field map, the declared `searchableFields`, and the display field the + * search resolution falls back on. + */ +export interface StoredMetadataSearchSchema { + fields?: unknown; + searchableFields?: unknown; + nameField?: unknown; + displayNameField?: unknown; +} + +/** + * [#21207, #21544] A `search` on a stored-metadata table never scans its body or + * content-hash columns — the ONE narrowing the generic data door and the + * in-process reader-context seam (`@objectstack/runtime`) both call, so a + * search answers the same through either. + * + * A search is a substring filter the engine evaluates over every column it + * scans, and with no `searchableFields` declared it scans every text-like + * column — the stored body and both stored hashes among them. Over those it is + * the verifier the filter refusals close: a guessed hash, or a guessed prefix + * of withheld credential material, returns the row exactly when it is right. + * The authority on which columns a search may never scan is the family's + * search predicate ({@link storedMetadataSearchRefusal}), asked per field: + * + * - an EXPLICIT field list — `searchFields`, else the object-form + * `search.fields`, the engine's own precedence and both its shapes (a comma + * string or an array) — naming one is refused, `INVALID_FIELD` / 400, the + * evaluate refusals' envelope, under the caller's wire spelling of the slot + * (`wireSpelling`, the door's; the bare canonical name when absent); + * - a search that names none is NARROWED: the object's resolved searchable set + * ({@link resolveSearchFieldResolution}) minus those columns is returned, for + * the caller to run as `searchFields` — the engine intersects an override + * with that set and never widens it; + * - a set that narrows to nothing is refused rather than returned empty: an + * empty override is ABSENT to the engine, which would then scan the whole + * default set, these columns included. + * + * Returns `undefined` when the query runs as it is — an object outside the + * family, no search, an explicit list that passes, or no readable field map + * (the engine has none to expand a search over either; the same rule the door's + * field gates apply: a non-object, array or empty `fields`). Throws the refusal. + */ +export function narrowStoredMetadataSearch( + object: string, + query: Readonly>, + schema: StoredMetadataSearchSchema | null | undefined, + wireSpelling: Readonly<{ search?: string; searchFields?: string }> = {}, +): string[] | undefined { + if (!isStoredMetadataBodyObject(object)) return undefined; + const search = query.search; + const objectForm = search !== null && typeof search === 'object'; + const objectFormFields = objectForm ? (search as Record).fields : undefined; + const [explicit, param] = query.searchFields != null + ? [query.searchFields, wireSpelling.searchFields ?? 'searchFields'] + : objectFormFields != null + ? [objectFormFields, wireSpelling.search ?? 'search'] + : [undefined, '']; + const names: string[] = typeof explicit === 'string' + ? explicit.split(',').map((s) => s.trim()).filter(Boolean) + : Array.isArray(explicit) ? explicit.filter((f: unknown): f is string => typeof f === 'string') : []; + if (names.length > 0) { + const refusal = storedMetadataSearchRefusal(object, names, param); + if (refusal) throw refusal; + return undefined; + } + if (search == null) return undefined; + const fields = schema?.fields; + if (!fields || typeof fields !== 'object' || Array.isArray(fields) || Object.keys(fields).length === 0) { + return undefined; + } + const { allowed } = resolveSearchFieldResolution({ + fields: fields as Record, + searchableFields: schema?.searchableFields as string[] | undefined, + displayField: (schema?.nameField ?? schema?.displayNameField) as string | undefined, + }); + const searchParam = wireSpelling.search ?? 'search'; + const narrowed = allowed.filter((field) => !storedMetadataSearchRefusal(object, [field], searchParam)); + if (narrowed.length === 0) { + const refusal = storedMetadataSearchRefusal(object, allowed, searchParam); + if (refusal) throw refusal; + return undefined; + } + return narrowed; +} + /** * Service Configuration for Discovery * Maps service names to their routes and plugin providers. @@ -11510,62 +11711,6 @@ export class ObjectStackProtocolImplementation implements throw err; } - /** - * [#21207] A `search` on a stored-metadata table never scans its body or - * content-hash columns ({@link STORED_METADATA_UNSEARCHABLE_COLUMNS}). - * - * A search is a substring filter the engine evaluates over every column it - * scans, and with no `searchableFields` declared it scans every text-like - * column — the stored body and both stored hashes among them. Over those - * it is the verifier the filter refusals close: a guessed hash, or a guessed - * prefix of withheld credential material, returns the row exactly when it - * is right. So an explicit field list naming one is refused - * (`INVALID_FIELD` / 400, the evaluate refusals' envelope), and a search - * that names none is handed to the engine with the object's searchable set - * minus those columns — the engine intersects an override with that set and - * never widens it. Runs after {@link assertSearchFieldsAreSearchable}, so a - * name that is not searchable at all keeps its own answer. - */ - private narrowStoredMetadataSearch( - object: string, - options: Record, - wireSpelling: Record, - ): void { - if (!isStoredMetadataBodyObject(object)) return; - const objectForm = options.search !== null && typeof options.search === 'object'; - const [explicit, param] = options.searchFields != null - ? [options.searchFields, wireSpelling.searchFields ?? 'searchFields'] - : objectForm && options.search.fields != null - ? [options.search.fields, wireSpelling.search ?? 'search'] - : [undefined, '']; - const names: string[] = typeof explicit === 'string' - ? explicit.split(',').map((s: string) => s.trim()).filter(Boolean) - : Array.isArray(explicit) ? explicit.filter((f: unknown): f is string => typeof f === 'string') : []; - if (names.length > 0) { - const refusal = storedMetadataSearchRefusal(object, names, param); - if (refusal) throw refusal; - return; - } - if (options.search == null) return; - const gate = this.resolveQueryFields(object); - // No field map: the engine has none to expand a search over either. - if (!gate) return; - const { allowed } = resolveSearchFieldResolution({ - fields: gate.fields, - searchableFields: gate.schema?.searchableFields, - displayField: gate.schema?.nameField ?? gate.schema?.displayNameField, - }); - const narrowed = allowed.filter((field) => !STORED_METADATA_UNSEARCHABLE_COLUMNS.includes(field)); - if (narrowed.length === 0) { - // An empty override is ABSENT to the engine, which would then scan - // the whole default set — these columns included. Refuse instead. - const refusal = storedMetadataSearchRefusal(object, allowed, wireSpelling.search ?? 'search'); - if (refusal) throw refusal; - return; - } - options.searchFields = narrowed; - } - /** * [#4254] GROUP-BY axis. A grouping target the object does not have is * refused (`400 INVALID_FIELD`); a grouping target the spec cannot read is @@ -12096,7 +12241,13 @@ export class ObjectStackProtocolImplementation implements } // [#21207] …and on a stored-metadata table a search never scans the body // or content-hash columns: refused when named, narrowed away otherwise. - this.narrowStoredMetadataSearch(request.object, options, wireSpelling); + // Runs after the searchability gate above, so a name that is not + // searchable at all keeps its own answer. [#21544] The ONE narrowing, + // a module function the in-process reader-context seam calls too. + const narrowedSearchFields = narrowStoredMetadataSearch( + request.object, options, this.engine?.registry?.getObject?.(request.object), wireSpelling, + ); + if (narrowedSearchFields) options.searchFields = narrowedSearchFields; // Boolean fields for (const key of ['distinct', 'count']) { @@ -12215,14 +12366,14 @@ export class ObjectStackProtocolImplementation implements // evaluates the body — a filter oracle that rebuilds a withheld // credential by probing, or an order over the same bytes — so it is // refused here, in the same shape as the grouping refusal, before the - // engine is asked. Field keys are collected the same way - // `assertFilterFieldsExist` reads them, so a nested-relation filter whose - // HEAD segment is the body column is caught too. - const aggregationFilterFields = Array.isArray(options.aggregations) - ? (options.aggregations as ReadonlyArray<{ filter?: unknown }>).flatMap((a) => - collectFilterFieldKeys(a?.filter)) - : []; - const filterFields = [...collectFilterFieldKeys(options.where), ...aggregationFilterFields]; + // engine is asked. [#21544] The columns a filter READS are collected by + // the family's ONE collector, which the reader-context seam calls too: + // every key's head AND every cross-field `{ $field }` comparand, at any + // depth, in `where` and each `aggregations[i].filter` — so a filter that + // reads the body or a hash without naming it as a key (a comparand), or + // names it below the ingress collector's depth backstop, is refused like + // a direct one. `[]` outside the family. + const filterFields = collectStoredMetadataFilterFields(request.object, options); const sortFields = Array.isArray(options.orderBy) ? (options.orderBy as ReadonlyArray<{ field?: unknown }>).map((e) => e?.field) : []; diff --git a/packages/runtime/src/stored-metadata-reader-seam.ts b/packages/runtime/src/stored-metadata-reader-seam.ts index 27efc341299..51fe56bb5e7 100644 --- a/packages/runtime/src/stored-metadata-reader-seam.ts +++ b/packages/runtime/src/stored-metadata-reader-seam.ts @@ -37,14 +37,18 @@ * (`storedMetadataBodyGroupingRefusal`, `storedMetadataBodyPredicateRefusal`, * `storedMetadataHashEvaluateRefusal`, `storedMetadataSearchRefusal`, * `@objectstack/metadata-protocol`), in the door's own order — a copy of any - * of them here would be a second definition of which shapes leak. A `count` - * with such a predicate is an oracle too, so it is guarded the same way (it - * serves no row, so only the refusal applies to it). A default `$search` - * is NARROWED to the door's served set — the body and hash columns removed, - * judged field by field by the door's own search predicate — rather than - * refused, so a body may still search a family table by `name` exactly as - * the door serves it; a search that would scan nothing after the removal is - * refused. + * of them here would be a second definition of which shapes leak. What the + * predicates are fed is the door's own too (#21544): the columns a filter + * reads come from the door's ONE filter-field collector + * (`collectStoredMetadataFilterFields`: every key's head and every + * cross-field `{ $field }` comparand, at any depth), and a default `$search` + * is NARROWED by the door's ONE narrowing (`narrowStoredMetadataSearch`) — + * the body and hash columns removed, judged field by field by the door's own + * search predicate — rather than refused, so a body may still search a + * family table by `name` exactly as the door serves it; a search that would + * scan nothing after the removal is refused. A `count` with such a predicate + * is an oracle too, so it is guarded the same way and runs the guarded query + * (it serves no row, so only the refusal applies to it). * 2. **Serve the body projected and the hash keyed** ({@link serveStoredMetadataRead}), * using the door's `storedMetadataBodyProjection`, `redactStoredMetadataRows` * (the family's ONE redactor in `@objectstack/spec/kernel`) and @@ -75,18 +79,18 @@ import { isStoredMetadataBodyObject } from '@objectstack/spec/kernel'; import { storedMetadataBodyWriteRefusal } from './stored-metadata-body-boundary.js'; -import { isFilterAST, parseFilterAST, resolveSearchFieldResolution } from '@objectstack/spec/data'; -import { collectConditionFields } from '@objectstack/plugin-security'; import { + collectStoredMetadataFilterFields, ephemeralStoredHashDigest, + narrowStoredMetadataSearch, redactStoredMetadataRows, serveStoredMetadataHashColumnRows, storedMetadataBodyProjection, storedMetadataBodyGroupingRefusal, storedMetadataBodyPredicateRefusal, storedMetadataHashEvaluateRefusal, - storedMetadataSearchRefusal, type StoredHashDigest, + type StoredMetadataSearchSchema, } from '@objectstack/metadata-protocol'; /** @@ -106,27 +110,6 @@ function isPlainRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } -/** - * Every column a filter NAMES, structure discarded — lowering a `FilterArray` - * to a `FilterCondition` first so the array door a direct engine call still - * honours (`lowerWhereFilterArray`) is read the same as the object door. - * - * The walk is the generic data door's sibling, `collectConditionFields` - * (`@objectstack/plugin-security`): the door's own `collectFilterFieldKeys` is - * internal to `protocol.ts` and cannot be imported here, and restating it would - * be the second definition the family rule forbids. This collector gates on a - * dotted head and also reads a cross-field `{ $field }` comparand, so it refuses - * a DOTTED or COMPARAND reference to the body or hash columns the door's own - * collector would miss — strictly MORE than the door, never a legitimate - * scalar-column query, since the family columns are the only ones these - * predicates name. - */ -function filterHeadFields(where: unknown): string[] { - if (where == null) return []; - const lowered = isFilterAST(where) ? parseFilterAST(where) : where; - return [...collectConditionFields(lowered)]; -} - /** The fields an `orderBy` names (`[{ field }]`, or a bare string entry). */ function sortFieldsOf(orderBy: unknown): unknown[] { if (!Array.isArray(orderBy)) return []; @@ -134,73 +117,32 @@ function sortFieldsOf(orderBy: unknown): unknown[] { } /** - * Narrow or refuse a `$search` on a family table the way the data door's - * `narrowStoredMetadataSearch` does, consuming the door's own search predicate - * ({@link storedMetadataSearchRefusal}) as the authority on which columns a - * search may never scan: - * - * - an EXPLICIT field list (`searchFields`, or the object-form `search.fields`) - * naming an unscannable column is refused; - * - a DEFAULT search is narrowed to the resolved searchable set minus the - * columns the predicate refuses, field by field, and the query runs with that - * `searchFields`; a set that narrows to empty is refused. - * - * Returns the query the read should run — the same reference when nothing - * changed, a shallow copy carrying the narrowed `searchFields` otherwise. + * The object definition the default-`$search` narrowing reads, from the + * engine's registry — the registry the data door reads the same definition + * from. `undefined` when the engine exposes none. */ -function narrowFamilySearch(object: string, query: Record, engine: unknown): Record { - const search = query.search; - const objectForm = search !== null && typeof search === 'object'; - const explicitRaw = query.searchFields != null - ? query.searchFields - : objectForm ? (search as Record).fields : undefined; - const param = query.searchFields != null ? 'searchFields' : 'search'; - const names: string[] = typeof explicitRaw === 'string' - ? explicitRaw.split(',').map((s) => s.trim()).filter(Boolean) - : Array.isArray(explicitRaw) - ? explicitRaw.filter((f): f is string => typeof f === 'string') - : []; - if (names.length > 0) { - const refusal = storedMetadataSearchRefusal(object, names, param); - if (refusal) throw refusal; - return query; - } - if (search == null) return query; - const schema = typeof (engine as { getObject?: (n: string) => unknown })?.getObject === 'function' - ? (engine as { getObject: (n: string) => any }).getObject(object) +function searchSchemaOf(object: string, engine: unknown): StoredMetadataSearchSchema | undefined { + const getObject = (engine as { getObject?: (name: string) => unknown } | null | undefined)?.getObject; + return typeof getObject === 'function' + ? (getObject.call(engine, object) as StoredMetadataSearchSchema | undefined) : undefined; - const fields = schema?.fields; - if (!fields) return query; - const { allowed } = resolveSearchFieldResolution({ - fields, - searchableFields: schema?.searchableFields, - displayField: schema?.nameField ?? schema?.displayNameField, - }); - const narrowed = allowed.filter((field) => !storedMetadataSearchRefusal(object, [field], 'search')); - if (narrowed.length === 0) { - const refusal = storedMetadataSearchRefusal(object, allowed, 'search'); - if (refusal) throw refusal; - return query; - } - return { ...query, searchFields: narrowed }; } /** * Refuse every EVALUATE shape on a family read, in the data door's own order * (search, grouping, body filter / sort, hash filter / sort / grouping), each - * through the door's own predicate. Returns the query the read should run, - * which may carry a narrowed `$search` field set. A non-family object, and a - * query that is not a record, pass through untouched. + * through the door's own predicate, fed by the door's own narrowing and + * collector. Returns the query the read should run: the caller's own, or a + * shallow copy carrying the narrowed `searchFields` of a default `$search`. A + * non-family object, and a query that is not a record, pass through untouched. */ function refuseOrNarrowStoredMetadataEvaluate(object: string, query: unknown, engine: unknown): unknown { if (!isStoredMetadataBodyObject(object) || !isPlainRecord(query)) return query; - const next = narrowFamilySearch(object, query, engine); + const narrowedSearchFields = narrowStoredMetadataSearch(object, query, searchSchemaOf(object, engine)); + const next = narrowedSearchFields ? { ...query, searchFields: narrowedSearchFields } : query; const grouping = storedMetadataBodyGroupingRefusal(object, next.groupBy); if (grouping) throw grouping; - const aggregationFilterFields = Array.isArray(next.aggregations) - ? (next.aggregations as ReadonlyArray<{ filter?: unknown }>).flatMap((a) => filterHeadFields(a?.filter)) - : []; - const filterFields = [...filterHeadFields(next.where), ...filterHeadFields(next.filter), ...aggregationFilterFields]; + const filterFields = collectStoredMetadataFilterFields(object, next); const sortFields = sortFieldsOf(next.orderBy); const bodyPredicate = storedMetadataBodyPredicateRefusal(object, { filterFields, sortFields }); if (bodyPredicate) throw bodyPredicate; @@ -296,11 +238,10 @@ function serveRepository(objectName: string, repo: unknown, engine: unknown): un if (prop === COUNT_READ) { // `async` so a refusal leaves as a rejected promise, the shape every // other verb's refusal takes — `count` answers a number, nothing to - // serve, so only the evaluate guard runs. - return async (query?: unknown, ...rest: unknown[]) => { - refuseOrNarrowStoredMetadataEvaluate(objectName, query, engine); - return value.call(target, query, ...rest); - }; + // serve, so only the evaluate guard runs, and the count runs the query + // the guard returns, exactly as a row-serving read does. + return async (query?: unknown, ...rest: unknown[]) => + value.call(target, refuseOrNarrowStoredMetadataEvaluate(objectName, query, engine), ...rest); } if (WRITE_RETURN_VERBS.has(prop)) { // [#21454] Serve what the write RETURNS — for the contexts that may From efb9145ed12d71194c2fe032d1b07818ded878c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 16:46:26 +0000 Subject: [PATCH 2/6] test(metadata-protocol,runtime): pin the family collector and narrowing at the door, and door/seam parity (WIP) Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- ...-door-stored-metadata-filter-reads.test.ts | 232 ++++++++++++++++++ .../src/stored-metadata-reader-seam.test.ts | 170 ++++++++++++- 2 files changed, 401 insertions(+), 1 deletion(-) create mode 100644 packages/metadata-protocol/src/protocol.data-door-stored-metadata-filter-reads.test.ts diff --git a/packages/metadata-protocol/src/protocol.data-door-stored-metadata-filter-reads.test.ts b/packages/metadata-protocol/src/protocol.data-door-stored-metadata-filter-reads.test.ts new file mode 100644 index 00000000000..7ecb270e3ab --- /dev/null +++ b/packages/metadata-protocol/src/protocol.data-door-stored-metadata-filter-reads.test.ts @@ -0,0 +1,232 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21544] The stored-metadata family's ONE filter-field collector and ONE + * default-search narrowing — the two module functions the generic data door + * calls and exports, so the in-process reader-context seam + * (`@objectstack/runtime`) calls the same two (the door / seam parity table + * lives beside the seam, `stored-metadata-reader-seam.test.ts`). + * + * The door's collector used to be the ingress key collector, and the + * measurement on this card found two shapes that READ a family column at the + * generic data door without the family's refusal ever seeing it, on both + * `sys_metadata` and `sys_metadata_history`, over HTTP and in-process alike: + * + * - a cross-field comparand (`{ name: { $ne: { $field: 'metadata' } } }`), + * which the SQL drivers evaluate row by row — the comparison partitioned + * the rows by the stored body's value; + * - a direct predicate nested below the key collector's 32-level depth + * backstop, which ran as written: a body `$contains` of the stored + * credential answered the row and a wrong guess answered none, on + * driver-sqlite-wasm and driver-memory — a credential oracle. + * + * Both are refused now, before the engine is asked, in the family's envelope. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { SysMetadataHistoryObject, SysMetadataObject } from '@objectstack/metadata-core'; +import { + collectStoredMetadataFilterFields, + narrowStoredMetadataSearch, + ObjectStackProtocolImplementation, +} from './protocol.js'; + +/** `n` nested `$and` levels above `leaf`: the leaf sits at depth `n`. */ +const deep = (n: number, leaf: Record): Record => + (n === 0 ? leaf : { $and: [deep(n - 1, leaf)] }); + +const sorted = (names: readonly string[]) => [...names].sort(); + +describe('[#21544] collectStoredMetadataFilterFields — every column a read query\'s filters read', () => { + const cases: Array<[string, unknown, string[]]> = [ + ['a key', { where: { type: 'view' } }, ['type']], + ['a dotted key reads its head', { where: { 'metadata.config': 'x' } }, ['metadata']], + ['a cross-field comparand', { where: { name: { $eq: { $field: 'metadata' } } } }, ['metadata', 'name']], + ['a reference as the implicit-equality comparand', { where: { name: { $field: 'checksum' } } }, ['checksum', 'name']], + ['a reference in a list', { where: { name: { $in: [{ $field: 'checksum' }] } } }, ['checksum', 'name']], + [ + 'a reference in an addDays offset', + { where: { created_at: { $lte: { $field: 'created_at', addDays: { $field: 'previous_checksum' } } } } }, + ['created_at', 'previous_checksum'], + ], + ['a dotted reference reads its head', { where: { name: { $ne: { $field: 'metadata.x' } } } }, ['metadata', 'name']], + [ + 'under $and / $or / $not', + { where: { $or: [{ $not: { name: { $lt: { $field: 'change_note' } } } }, { $and: [{ type: 'x' }] }] } }, + ['change_note', 'name', 'type'], + ], + ['under an unrecognised $ key', { where: { $nor: [{ metadata: 'x' }] } }, ['metadata']], + [ + 'a nested-relation condition: its keys are another object\'s, a reference beneath it is read', + { where: { organization_id: { metadata: 'x', name: { $eq: { $field: 'checksum' } } } } }, + ['checksum', 'organization_id'], + ], + ['at any depth: a key 40 levels down', { where: deep(40, { metadata: { $contains: 'z' } }) }, ['metadata']], + ['at any depth: a reference 40 levels down', { where: deep(40, { name: { $ne: { $field: 'checksum' } } }) }, ['checksum', 'name']], + ['a FilterArray is lowered first', { where: [['metadata', 'contains', 'z']] }, ['metadata']], + ['the engine\'s `filter` alias', { filter: { checksum: 'x' } }, ['checksum']], + [ + 'each aggregation filter', + { aggregations: [{ function: 'count', alias: 'n', filter: { name: { $ne: { $field: 'metadata' } } } }] }, + ['metadata', 'name'], + ], + [ + 'not `having`: its names are the aggregated row\'s, an alias spelled like a family column included', + { groupBy: ['type'], aggregations: [{ function: 'count', alias: 'metadata' }], having: { metadata: { $gt: 0 } } }, + [], + ], + ]; + for (const [label, query, expected] of cases) { + it(label, () => { + expect(sorted(collectStoredMetadataFilterFields('sys_metadata', query))).toEqual(expected); + expect(sorted(collectStoredMetadataFilterFields('sys_metadata_history', query))).toEqual(expected); + }); + } + + it('[] for an object outside the family, and for a query that is not a record', () => { + expect(collectStoredMetadataFilterFields('file_blob', { where: { metadata: 'x' } })).toEqual([]); + expect(collectStoredMetadataFilterFields('sys_metadata', null)).toEqual([]); + expect(collectStoredMetadataFilterFields('sys_metadata', [{ metadata: 'x' }])).toEqual([]); + }); + + it('terminates on a self-referential live object, and reads a node shared as a comparand and a condition both ways', () => { + const cyclic: Record = { type: 'x' }; + cyclic.$and = [cyclic]; + expect(collectStoredMetadataFilterFields('sys_metadata', { where: cyclic })).toEqual(['type']); + const shared = { metadata: 'x' }; + expect(sorted(collectStoredMetadataFilterFields('sys_metadata', { where: { name: { $eq: shared }, $and: [shared] } }))) + .toEqual(['metadata', 'name']); + }); +}); + +describe('[#21544] narrowStoredMetadataSearch — the one default-search narrowing', () => { + function refusalOf(run: () => unknown): any { + try { + run(); + } catch (e) { + return e; + } + throw new Error('expected a refusal, but the search was not refused'); + } + + it('an explicit list naming the body or a hash column is refused, under the slot the caller used', () => { + for (const [query, wire, param, field] of [ + [{ search: 'z', searchFields: ['name', 'metadata'] }, {}, 'searchFields', 'metadata'], + [{ search: 'z', searchFields: 'name, checksum' }, {}, 'searchFields', 'checksum'], + [{ search: { query: 'z', fields: ['previous_checksum'] } }, {}, 'search', 'previous_checksum'], + [{ search: 'z', searchFields: 'change_note' }, { searchFields: '$searchFields' }, '$searchFields', 'change_note'], + ] as const) { + const err = refusalOf(() => narrowStoredMetadataSearch('sys_metadata_history', query, SysMetadataHistoryObject, wire)); + expect(err).toMatchObject({ code: 'INVALID_FIELD', status: 400, param, field, object: 'sys_metadata_history' }); + } + }); + + it('an explicit list that names neither runs as it is', () => { + expect(narrowStoredMetadataSearch('sys_metadata', { search: 'z', searchFields: ['name'] }, SysMetadataObject)).toBeUndefined(); + }); + + it('a default search is narrowed to the searchable set without the body and hash columns', () => { + for (const [object, schema, unscannable] of [ + ['sys_metadata', SysMetadataObject, ['metadata', 'checksum']], + ['sys_metadata_history', SysMetadataHistoryObject, ['metadata', 'checksum', 'previous_checksum', 'change_note']], + ] as const) { + const narrowed = narrowStoredMetadataSearch(object, { search: 'z' }, schema); + expect(narrowed).toContain('name'); + for (const column of unscannable) expect(narrowed).not.toContain(column); + } + }); + + it('a default search whose set narrows to nothing is refused, never handed on empty', () => { + const onlyFamily = { ...SysMetadataObject, searchableFields: ['metadata', 'checksum'] }; + const err = refusalOf(() => narrowStoredMetadataSearch('sys_metadata', { search: 'z' }, onlyFamily, { search: '$search' })); + expect(err).toMatchObject({ code: 'INVALID_FIELD', status: 400, param: '$search', field: 'metadata' }); + }); + + it('runs as it is outside the family, without a search, and without a readable field map', () => { + expect(narrowStoredMetadataSearch('file_blob', { search: 'z', searchFields: ['metadata'] }, SysMetadataObject)).toBeUndefined(); + expect(narrowStoredMetadataSearch('sys_metadata', { where: { type: 'view' } }, SysMetadataObject)).toBeUndefined(); + for (const schema of [undefined, {}, { fields: [] }, { fields: {} }]) { + expect(narrowStoredMetadataSearch('sys_metadata', { search: 'z' }, schema)).toBeUndefined(); + } + }); +}); + +describe('[#21544] data door — a filter that reads the body or a hash without naming it as a key is refused', () => { + const SCHEMAS: Record = { + sys_metadata: SysMetadataObject, + sys_metadata_history: SysMetadataHistoryObject, + file_blob: { name: 'file_blob', fields: { name: { name: 'name', type: 'text' }, checksum: { name: 'checksum', type: 'text' } } }, + }; + + function makeProtocol() { + const find = vi.fn(async () => []); + const aggregate = vi.fn(async () => []); + const count = vi.fn(async () => 0); + const engine: any = { registry: { getObject: (n: string) => SCHEMAS[n] }, find, findOne: vi.fn(async () => null), count, aggregate }; + return { p: new ObjectStackProtocolImplementation(engine), find, aggregate, count }; + } + + async function refusal(run: () => Promise): Promise { + try { + await run(); + } catch (e) { + return e; + } + throw new Error('expected a refusal, but the query ran'); + } + + const shapes: Array<[string, string, (column: string) => Record]> = [ + ['a cross-field comparand in `where`', 'filter', (c) => ({ where: { name: { $ne: { $field: c } } } })], + ['a cross-field comparand under $not', 'filter', (c) => ({ where: { $not: { type: { $lt: { $field: c } } } } })], + ['a direct predicate 33 levels deep', 'filter', (c) => ({ where: deep(33, { [c]: { $contains: 'guess' } }) })], + ['a cross-field comparand 33 levels deep', 'filter', (c) => ({ where: deep(33, { name: { $eq: { $field: c } } }) })], + [ + 'a cross-field comparand in an aggregation filter', + 'filter', + (c) => ({ groupBy: ['type'], aggregations: [{ function: 'count', alias: 'n', filter: { name: { $ne: { $field: c } } } }] }), + ], + [ + 'a direct predicate 33 levels deep in an aggregation filter', + 'filter', + (c) => ({ groupBy: ['type'], aggregations: [{ function: 'count', alias: 'n', filter: deep(33, { [c]: { $contains: 'guess' } }) }] }), + ], + ]; + const columns: Array<[string, string]> = [ + ['sys_metadata', 'metadata'], + ['sys_metadata', 'checksum'], + ['sys_metadata_history', 'metadata'], + ['sys_metadata_history', 'previous_checksum'], + ['sys_metadata_history', 'change_note'], + ]; + for (const [label, param, build] of shapes) { + for (const [object, column] of columns) { + it(`${label}, reading '${column}' on '${object}': INVALID_FIELD / 400, and the engine is never asked`, async () => { + const { p, find, aggregate, count } = makeProtocol(); + const err = await refusal(() => p.findData({ object, query: build(column) })); + expect(err).toMatchObject({ code: 'INVALID_FIELD', status: 400, param, field: column, object }); + expect(find).not.toHaveBeenCalled(); + expect(aggregate).not.toHaveBeenCalled(); + expect(count).not.toHaveBeenCalled(); + }); + } + } + + it('control: the same shapes over scalar columns of a family table still run', async () => { + const { p, find, aggregate } = makeProtocol(); + await p.findData({ object: 'sys_metadata', query: { where: { name: { $ne: { $field: 'type' } } } } }); + await p.findData({ object: 'sys_metadata', query: { where: deep(33, { type: 'view' }) } }); + expect(find).toHaveBeenCalledTimes(2); + await p.findData({ + object: 'sys_metadata', + query: { groupBy: ['type'], aggregations: [{ function: 'count', alias: 'n', filter: { name: { $ne: { $field: 'type' } } } }] }, + }); + expect(aggregate).toHaveBeenCalledTimes(1); + }); + + it('control: an object outside the family reads its own `checksum` through a comparand and at depth', async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'file_blob', query: { where: { name: { $ne: { $field: 'checksum' } } } } }); + await p.findData({ object: 'file_blob', query: { where: deep(33, { checksum: 'sha256:0000' }) } }); + expect(find).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/runtime/src/stored-metadata-reader-seam.test.ts b/packages/runtime/src/stored-metadata-reader-seam.test.ts index 59ea232dbd7..3e3f06745b4 100644 --- a/packages/runtime/src/stored-metadata-reader-seam.test.ts +++ b/packages/runtime/src/stored-metadata-reader-seam.test.ts @@ -14,8 +14,10 @@ import { assertEngineFindOnePredicate, assertEngineUpdateDispatch, assertEngineDeleteDispatch, + SysMetadataHistoryObject, + SysMetadataObject, } from '@objectstack/metadata-core'; -import { ephemeralStoredHashDigest } from '@objectstack/metadata-protocol'; +import { ephemeralStoredHashDigest, ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; import { serveStoredMetadataRead, serveStoredMetadataReadsThrough } from './stored-metadata-reader-seam.js'; const SENTINEL = 'seam-unit-sentinel-41d7'; @@ -265,3 +267,169 @@ describe('[#21454] the engine action verb is never on a served body\'s surface', expect((api.object('sys_metadata') as any).execute).toBeUndefined(); }); }); + +/** + * [#21544] The door and the seam answer every family filter and search the + * SAME way, because both call the door's two exported functions — its one + * filter-field collector (`collectStoredMetadataFilterFields`) and its one + * default-search narrowing (`narrowStoredMetadataSearch`). One table, run + * through both: the generic data door (`findData`, over an engine double whose + * registry answers the real family definitions) and the seam (a scoped API + * served through `serveStoredMetadataReadsThrough`, whose engine face answers + * the same definitions). A future divergence fails one shared case. + * + * An outcome is the refusal's `code`, `status` and the column it names (the + * door's earlier ingress gate answers a DOTTED key first, naming the whole + * key under `where`, so the column is compared by its head and `param` is + * compared on the search rows only), or — for a query that ran — the + * `searchFields` the read was handed. + */ +describe('[#21544] door / seam parity — one collector, one narrowing', () => { + type Outcome = + | { refused: { code: unknown; status: unknown; column: string; param?: unknown } } + | { ran: { searchFields: unknown } }; + const deep = (n: number, leaf: Record): Record => + (n === 0 ? leaf : { $and: [deep(n - 1, leaf)] }); + const familySchemas = (overrides: Record = {}): Record => ({ + sys_metadata: SysMetadataObject, + sys_metadata_history: SysMetadataHistoryObject, + ...overrides, + }); + const refusedAs = (e: any, withParam: boolean): Outcome => ({ + refused: { + code: e?.code, + status: e?.status, + column: String(e?.field).split('.')[0] as string, + ...(withParam ? { param: e?.param } : {}), + }, + }); + const isAggregate = (query: Record) => 'groupBy' in query || 'aggregations' in query; + + async function viaDoor(object: string, query: Record, schemas: Record, withParam: boolean): Promise { + const handed: Array> = []; + const record = async (_object: string, options: Record) => { + handed.push(options); + return []; + }; + const engine: any = { + registry: { getObject: (name: string) => schemas[name] }, + find: record, + aggregate: record, + findOne: async () => null, + count: async () => 0, + getKeyedDigest: () => provider, + }; + try { + await new ObjectStackProtocolImplementation(engine).findData({ object, query: structuredClone(query) }); + } catch (e) { + return refusedAs(e, withParam); + } + return { ran: { searchFields: handed[0]?.searchFields } }; + } + + async function viaSeam(object: string, query: Record, schemas: Record, withParam: boolean): Promise { + const handed: Array> = []; + const repo = { + async find(q: Record) { handed.push(q); return []; }, + async aggregate(q: Record) { handed.push(q); return []; }, + }; + const api = serveStoredMetadataReadsThrough( + { object: (_name: string) => repo } as any, + { getKeyedDigest: () => provider, getObject: (name: string) => schemas[name] }, + ); + const served = api.object(object) as typeof repo; + try { + await (isAggregate(query) ? served.aggregate(structuredClone(query)) : served.find(structuredClone(query))); + } catch (e) { + return refusedAs(e, withParam); + } + return { ran: { searchFields: handed[0]?.searchFields } }; + } + + // [label, object, query, expected column (refused) or null (ran), compare param?, schema overrides] + const cases: Array<[string, string, Record, string | null, boolean?, Record?]> = [ + // The search half: an explicit list, the default search, an emptied set. + ['an explicit search list (array) naming the body', 'sys_metadata', { search: 'z', searchFields: ['name', 'metadata'] }, 'metadata', true], + ['an explicit search list (comma string) naming a hash', 'sys_metadata_history', { search: 'z', searchFields: 'name,checksum' }, 'checksum', true], + ['an explicit object-form search list naming the parent hash', 'sys_metadata_history', { search: { query: 'z', fields: ['previous_checksum'] } }, 'previous_checksum', true], + ['an explicit search list naming neither', 'sys_metadata', { search: 'z', searchFields: ['name'] }, null, true], + ['the default search (narrowed, then run)', 'sys_metadata', { search: 'datasource' }, null, true], + ['the default search on the history table', 'sys_metadata_history', { search: 'datasource' }, null, true], + [ + 'a default search whose set narrows to nothing', + 'sys_metadata', + { search: 'z' }, + 'metadata', + true, + { sys_metadata: { ...SysMetadataObject, searchableFields: ['metadata', 'checksum'] } }, + ], + // The collector half: what the measurement found the door's old collector missed, and its neighbours. + ['a direct body filter', 'sys_metadata', { where: { metadata: { $contains: 'z' } } }, 'metadata'], + ['a dotted body key', 'sys_metadata', { where: { 'metadata.config': 'z' } }, 'metadata'], + ['a dotted hash key', 'sys_metadata_history', { where: { 'checksum.x': 'z' } }, 'checksum'], + ['a cross-field comparand on the body', 'sys_metadata', { where: { name: { $eq: { $field: 'metadata' } } } }, 'metadata'], + ['a cross-field comparand on a hash', 'sys_metadata', { where: { type: { $lt: { $field: 'checksum' } } } }, 'checksum'], + ['a cross-field comparand on the change note', 'sys_metadata_history', { where: { name: { $ne: { $field: 'change_note' } } } }, 'change_note'], + ['a dotted cross-field reference', 'sys_metadata', { where: { name: { $ne: { $field: 'metadata.x' } } } }, 'metadata'], + ['a reference in a list', 'sys_metadata', { where: { name: { $in: [{ $field: 'checksum' }] } } }, 'checksum'], + ['a reference under $not', 'sys_metadata_history', { where: { $not: { name: { $eq: { $field: 'previous_checksum' } } } } }, 'previous_checksum'], + ['a direct body filter 33 levels deep', 'sys_metadata', { where: deep(33, { metadata: { $contains: 'z' } }) }, 'metadata'], + ['a cross-field comparand 33 levels deep', 'sys_metadata_history', { where: deep(33, { name: { $ne: { $field: 'checksum' } } }) }, 'checksum'], + [ + 'a cross-field comparand in an aggregation filter', + 'sys_metadata', + { groupBy: ['type'], aggregations: [{ function: 'count', alias: 'n', filter: { name: { $ne: { $field: 'metadata' } } } }] }, + 'metadata', + ], + [ + 'a body filter 33 levels deep in an aggregation filter', + 'sys_metadata_history', + { groupBy: ['type'], aggregations: [{ function: 'count', alias: 'n', filter: deep(33, { metadata: { $contains: 'z' } }) }] }, + 'metadata', + ], + // Controls: scalar columns, and a `having` alias spelled like a family column, run on both. + ['control: a cross-field comparand between scalar columns', 'sys_metadata', { where: { name: { $ne: { $field: 'type' } } } }, null], + ['control: a scalar filter 33 levels deep', 'sys_metadata', { where: deep(33, { type: 'view' }) }, null], + [ + 'control: a `having` on an aggregation alias spelled like the body column', + 'sys_metadata', + { groupBy: ['type'], aggregations: [{ function: 'count', alias: 'metadata' }], having: { metadata: { $gt: 0 } } }, + null, + ], + ]; + + for (const [label, object, query, column, withParam = false, overrides] of cases) { + it(`${label}: the door and the seam answer identically`, async () => { + const schemas = familySchemas(overrides); + const door = await viaDoor(object, query, schemas, withParam); + const seam = await viaSeam(object, query, schemas, withParam); + expect(seam).toEqual(door); + if (column === null) { + expect('ran' in door, `${label}: the door refused a query that should run`).toBe(true); + } else { + expect(door).toMatchObject({ refused: { code: 'INVALID_FIELD', status: 400, column } }); + } + }); + } + + it('the default search is handed on narrowed — the body and hash columns never scanned', async () => { + const door = await viaDoor('sys_metadata_history', { search: 'datasource' }, familySchemas(), false); + const searchFields = (door as { ran: { searchFields: string[] } }).ran.searchFields; + expect(searchFields).toContain('name'); + for (const column of ['metadata', 'checksum', 'previous_checksum', 'change_note']) expect(searchFields).not.toContain(column); + }); + + it('`count` runs the query the guard returns: a default search arrives narrowed', async () => { + const counted: Array> = []; + const repo = { async count(q: Record) { counted.push(q); return 0; } }; + const api = serveStoredMetadataReadsThrough( + { object: (_name: string) => repo } as any, + { getKeyedDigest: () => provider, getObject: (name: string) => familySchemas()[name] }, + ); + await (api.object('sys_metadata') as typeof repo).count({ search: 'datasource' }); + const searchFields = counted[0]?.searchFields as string[]; + expect(searchFields).toContain('name'); + expect(searchFields).not.toContain('metadata'); + expect(searchFields).not.toContain('checksum'); + }); +}); From 9be38c59877e404c58926435e2d8a37b198fdf27 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 16:47:31 +0000 Subject: [PATCH 3/6] chore(changeset): 21544 door narrowing export (minor metadata-protocol, patch runtime) Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- .changeset/21544-door-narrowing-export.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .changeset/21544-door-narrowing-export.md diff --git a/.changeset/21544-door-narrowing-export.md b/.changeset/21544-door-narrowing-export.md new file mode 100644 index 00000000000..353318eef41 --- /dev/null +++ b/.changeset/21544-door-narrowing-export.md @@ -0,0 +1,22 @@ +--- +'@objectstack/metadata-protocol': minor +'@objectstack/runtime': patch +--- + +fix(metadata-protocol)!: the generic data door refuses a stored-metadata filter that reads the body or a content hash through a cross-field comparand or below its depth backstop, and exports its one filter-field collector and one search narrowing for the reader-context seam (#21544) + +Clause-②: yes (narrowing) + + + +**BREAKING**: this narrows what the generic data door (`GET /api/v1/data/:object`, `POST /api/v1/data/:object/query` and the in-process `findData`) accepts when it reads `sys_metadata` or `sys_metadata_history`. Two filter shapes read the stored body column or a content-hash column (`checksum`, `previous_checksum`, or the history table's `change_note`) without the family's refusal ever seeing them, and both ran before this release: + +- a cross-field comparand naming one of those columns — `{ "name": { "$ne": { "$field": "metadata" } } }`, in `where` or in an aggregation's `filter`, under `$not` included. The SQL drivers evaluate it row by row, so row presence disclosed the column's value; +- a filter on one of those columns nested more than 32 combinators deep, which the door's field collector stopped reading at. A body `$contains` of a stored credential answered the row and a wrong guess answered none. + +Both now answer the door's `400 INVALID_FIELD`, naming the column, before the query runs — the answer the same filter already gets when it names the column directly. The route: filter those tables by their scalar columns (the type, the name, the state and the like), compare scalar columns with each other, and read the bodies with a plain list, which is served projected. Every other column of the two tables, and every other object, is unchanged; a dotted key into one of those columns was, and stays, refused by the door's dotted-path rule. It ships as `minor` under the launch-window convention for accept-set narrowings. + +- **`@objectstack/metadata-protocol`** exports two module functions the generic data door now calls itself: + - `collectStoredMetadataFilterFields(object, query)` — the family's one filter-field collector: every column a read query's filters read (`where`, the engine's `filter` alias and each aggregation filter): each key's head and each cross-field `{ $field }` comparand, at any depth. `[]` outside the family. + - `narrowStoredMetadataSearch(object, query, schema, wireSpelling?)` — the family's one default-search narrowing: an explicit search-field list naming the body or a hash column is refused, a default search is narrowed to the searchable set without them (returned for the caller to run as `searchFields`), and a set that narrows to nothing is refused. The `StoredMetadataSearchSchema` type it reads is exported beside it. +- **`@objectstack/runtime`**: the stored-metadata reader-context seam (`ctx.api.object(...)` for action and hook bodies, a handler's `ctx.api`, and `ctx.engine.find`) calls those two functions instead of its own copy of the narrowing and `@objectstack/plugin-security`'s condition walk, so the seam and the door answer every family filter and search identically. A `count` through the seam now runs the query the guard returns. The seam's accept set is unchanged: every shape it refused before it still refuses, now through the door's collector. From be99ceee6a24e3d720ff34c4f9247b4e541d5a81 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 17:01:32 +0000 Subject: [PATCH 4/6] test(runtime): parity case for an unrecognised $ key wrapping a body filter Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- packages/runtime/src/stored-metadata-reader-seam.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/runtime/src/stored-metadata-reader-seam.test.ts b/packages/runtime/src/stored-metadata-reader-seam.test.ts index 3e3f06745b4..ed3d6786740 100644 --- a/packages/runtime/src/stored-metadata-reader-seam.test.ts +++ b/packages/runtime/src/stored-metadata-reader-seam.test.ts @@ -373,6 +373,7 @@ describe('[#21544] door / seam parity — one collector, one narrowing', () => { ['a dotted cross-field reference', 'sys_metadata', { where: { name: { $ne: { $field: 'metadata.x' } } } }, 'metadata'], ['a reference in a list', 'sys_metadata', { where: { name: { $in: [{ $field: 'checksum' }] } } }, 'checksum'], ['a reference under $not', 'sys_metadata_history', { where: { $not: { name: { $eq: { $field: 'previous_checksum' } } } } }, 'previous_checksum'], + ['an unrecognised $ key wrapping a body filter', 'sys_metadata', { where: { $nor: [{ metadata: { $contains: 'z' } }] } }, 'metadata'], ['a direct body filter 33 levels deep', 'sys_metadata', { where: deep(33, { metadata: { $contains: 'z' } }) }, 'metadata'], ['a cross-field comparand 33 levels deep', 'sys_metadata_history', { where: deep(33, { name: { $ne: { $field: 'checksum' } } }) }, 'checksum'], [ From 45b0f990a33d11e6dace175cc7b267778e6de24b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 17:34:16 +0000 Subject: [PATCH 5/6] test(metadata-protocol,runtime): the door doubles carry no findOne (findData never reads one) Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- .../protocol.data-door-stored-metadata-filter-reads.test.ts | 4 +++- packages/runtime/src/stored-metadata-reader-seam.test.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.data-door-stored-metadata-filter-reads.test.ts b/packages/metadata-protocol/src/protocol.data-door-stored-metadata-filter-reads.test.ts index 7ecb270e3ab..0fdd5a33c32 100644 --- a/packages/metadata-protocol/src/protocol.data-door-stored-metadata-filter-reads.test.ts +++ b/packages/metadata-protocol/src/protocol.data-door-stored-metadata-filter-reads.test.ts @@ -162,7 +162,9 @@ describe('[#21544] data door — a filter that reads the body or a hash without const find = vi.fn(async () => []); const aggregate = vi.fn(async () => []); const count = vi.fn(async () => 0); - const engine: any = { registry: { getObject: (n: string) => SCHEMAS[n] }, find, findOne: vi.fn(async () => null), count, aggregate }; + // `findData` reads through find / count / aggregate only — no `findOne` + // on this double, so a door that started calling it would fail loudly. + const engine: any = { registry: { getObject: (n: string) => SCHEMAS[n] }, find, count, aggregate }; return { p: new ObjectStackProtocolImplementation(engine), find, aggregate, count }; } diff --git a/packages/runtime/src/stored-metadata-reader-seam.test.ts b/packages/runtime/src/stored-metadata-reader-seam.test.ts index ed3d6786740..85de3bd6ce9 100644 --- a/packages/runtime/src/stored-metadata-reader-seam.test.ts +++ b/packages/runtime/src/stored-metadata-reader-seam.test.ts @@ -315,7 +315,7 @@ describe('[#21544] door / seam parity — one collector, one narrowing', () => { registry: { getObject: (name: string) => schemas[name] }, find: record, aggregate: record, - findOne: async () => null, + // `findData` reads through find / count / aggregate only — no `findOne` here. count: async () => 0, getKeyedDigest: () => provider, }; From 0ac5749662d7dddffd9dc011e0e71375d8d79451 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 17:44:39 +0000 Subject: [PATCH 6/6] docs(metadata-protocol): state which collector rules were measured reaches Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- packages/metadata-protocol/src/protocol.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index c2ba22dfd14..dd017352101 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -4013,8 +4013,12 @@ function collectFilterFieldKeys( * (`{ function: 'count', alias: 'metadata' }`) is a legitimate count. * * What counts as a read, and why it is more than the ingress gate's - * {@link collectFilterFieldKeys} collects — each rule a measured reach at the - * generic data door before this collector existed: + * {@link collectFilterFieldKeys} collects. The first two rules are measured + * reaches at the generic data door before this collector existed (a family + * column evaluated, unrefused); the rest are the stricter collector's rules, + * adopted whole so the door and the seam agree — where measured, the door + * already refused those shapes elsewhere (its dotted-path rule, the engine's + * filter doors), and now the family's own refusal answers them first: * * - **A cross-field comparand is a read.** `{ name: { $eq: { $field: 'metadata' } } }` * compares each row's `name` with its stored body, and the SQL drivers