From c09e94bff3b2ca70dad12adf955fa658d8b5f010 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 11:53:47 +0000 Subject: [PATCH 1/4] test(rest): pin a paged search's total at the public data door (measurement first) Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T Co-authored-by: Claude --- .../rest/src/list-search-total-door.test.ts | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 packages/rest/src/list-search-total-door.test.ts diff --git a/packages/rest/src/list-search-total-door.test.ts b/packages/rest/src/list-search-total-door.test.ts new file mode 100644 index 00000000000..a00b98e0958 --- /dev/null +++ b/packages/rest/src/list-search-total-door.test.ts @@ -0,0 +1,156 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #22790 — under a search, a paged list's `total` is the number of matching + * records, counted, never a page-local estimate. + * + * Before: `findData` skipped the COUNT whenever a `search` rode the query + * (`engine.count()` took `where` only) and reported `offset + page + 1`, so + * `?$top=25&$search=Line` over 75 matches answered `total: 26`, and the last + * page claimed `hasMore: true`. Now `engine.count()` takes the query's + * `search` / `searchFields` and expands them through the one ADR-0061 expander + * `find` runs, so `total` is the count of exactly the rows `find` would page. + * + * Real chain: `RestServer` route handler → `findData` → `ObjectQL` → sqlite + * `SqlDriver`. A middleware stands in for a member's read scope, so the count + * is shown to compose with an injected read filter exactly as the rows do. + * + * Fixture (90 rows): `title` holds "Line" on rows 0–59, `notes` on 60–74, and + * the `internal: true` column `hint` on 75–89 — so the default search matches + * 75, `searchFields: ['title']` 60, and a count that scanned `hint` would say + * 90. The member owns every third row (25 of the 75 matches, 30 of the 90). + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { RestServer } from './rest-server'; + +const OBJECT = 'ticket'; +const MEMBER = 'member_user'; +const ROWS = Array.from({ length: 90 }, (_, i) => ({ + id: `t${String(i).padStart(2, '0')}`, + title: i < 60 ? `Line ${i}` : `Row ${i}`, + notes: i >= 60 && i < 75 ? 'see Line' : 'n/a', + hint: i >= 75 ? 'Line secret' : 'x', + owner: i % 3 === 0 ? MEMBER : 'admin_user', +})); + +type Body = { records?: unknown[]; total?: number; hasMore?: boolean; code?: string }; +type Route = { method: string; path: string; handler: (req: unknown, res: unknown) => Promise }; + +let engine: ObjectQL; +let listRoute: Route; +let queryRoute: Route; +let principal = 'admin_user'; +/** The `where` each driver face last received for {@link OBJECT}. */ +const driverWhere: { find?: unknown; count?: unknown } = {}; + +beforeAll(async () => { + const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }); + for (const face of ['find', 'count'] as const) { + const native = (driver as any)[face].bind(driver); + (driver as any)[face] = (object: string, ast: any, options?: unknown) => { + if (object === OBJECT) driverWhere[face] = ast?.where; + return native(object, ast, options); + }; + } + engine = new ObjectQL(); + engine.registerDriver(driver as never, true); + await engine.init(); + engine.registry.registerObject({ + name: OBJECT, label: 'Ticket', systemFields: false, nameField: 'title', + fields: { + id: { name: 'id', type: 'text', primaryKey: true }, + title: { name: 'title', type: 'text' }, + notes: { name: 'notes', type: 'text' }, + hint: { name: 'hint', type: 'text', internal: true }, + owner: { name: 'owner', type: 'text' }, + }, + } as never); + await engine.syncSchemas(); + await engine.insert(OBJECT, ROWS); + // The member's read scope, injected the way the security middlewares do it. + engine.registerMiddleware(async (ctx, next) => { + if (ctx.ast && ctx.object === OBJECT && (ctx.context as any)?.userId === MEMBER) { + ctx.ast.where = ctx.ast.where ? { $and: [ctx.ast.where, { owner: MEMBER }] } : { owner: MEMBER }; + } + await next(); + }); + + const rest = new RestServer( + { get() {}, post() {}, put() {}, delete() {}, patch() {}, use() {}, listen: async () => {}, close: async () => {} } as never, + new ObjectStackProtocolImplementation(engine as never) as never, + { api: { requireAuth: false } } as never, + ); + (rest as any).resolveExecCtx = async () => ({ userId: principal }); + rest.registerRoutes(); + const routes = rest.getRoutes() as Route[]; + listRoute = routes.find((r) => r.method === 'GET' && r.path === '/api/v1/data/:object')!; + queryRoute = routes.find((r) => r.method === 'POST' && r.path === '/api/v1/data/:object/query')!; +}); + +afterAll(async () => { + try { await engine?.destroy(); } catch { /* best-effort */ } +}); + +async function call(route: Route, req: Record, as = 'admin_user'): Promise { + principal = as; + let status: number | undefined; + let body: Body = {}; + const res: any = { write: () => true, end() {}, header: () => res }; + res.status = (code: number) => { status = code; return res; }; + res.json = (b: Body) => { body = b; return res; }; + await route.handler({ params: { object: OBJECT }, headers: {}, ...req }, res); + expect(status, `door refused: ${JSON.stringify(body)}`).toBeUndefined(); + return body; +} +const list = (query: Record, as?: string) => call(listRoute, { method: 'GET', query }, as); +const post = (body: Record, as?: string) => call(queryRoute, { method: 'POST', body }, as); + +describe('[#22790] a paged search reports the counted total', () => { + // 75, not 90: neither the rows nor the count scan the `internal: true` column. + it('GET ?$top&$search: total is the match count on every page, and hasMore agrees', async () => { + const unpaged = await list({ $search: 'Line' }); + expect(unpaged.records).toHaveLength(75); + + const first = await list({ $top: '25', $search: 'Line' }); + expect(first).toMatchObject({ total: 75, hasMore: true }); + expect(first.records).toHaveLength(25); + + const last = await list({ $top: '25', $skip: '50', $search: 'Line' }); + expect(last).toMatchObject({ total: 75, hasMore: false }); + expect(last.records).toHaveLength(25); + }); + + it('POST …/query: the same, and a `searchFields` narrowing narrows the count with the rows', async () => { + expect(await post({ search: 'Line', limit: 25 })).toMatchObject({ total: 75, hasMore: true }); + + const narrowed = await post({ search: 'Line', searchFields: ['title'], limit: 25 }); + expect(narrowed).toMatchObject({ total: 60, hasMore: true }); + expect((await post({ search: 'Line', searchFields: ['title'] })).records).toHaveLength(60); + }); + + it('one expander: the driver COUNT receives the same `where` the driver FIND does', async () => { + await list({ $top: '25', $search: 'Line notes' }); + expect(driverWhere.count).toBeDefined(); + expect(driverWhere.count).toEqual(driverWhere.find); + }); + + it('a member counts only the matches it can read', async () => { + expect(await list({ $top: '10', $search: 'Line' }, MEMBER)).toMatchObject({ total: 25, hasMore: true }); + expect((await list({ $search: 'Line' }, MEMBER)).records).toHaveLength(25); + }); + + it('CONTROL — a paged list with no search is unchanged, for both principals', async () => { + expect(await list({ $top: '25' })).toMatchObject({ total: 90, hasMore: true }); + expect(await list({ $top: '10' }, MEMBER)).toMatchObject({ total: 30, hasMore: true }); + }); + + it('CONTROL — `$count=false` with a search still omits total', async () => { + const body = await list({ $top: '25', $search: 'Line', $count: 'false' }); + expect('total' in body).toBe(false); + expect(body.hasMore).toBe(true); + }); +}); From 7427192988b49c36bff9b4c7448fd31adc36ed1d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 12:05:36 +0000 Subject: [PATCH 2/4] fix(objectql,metadata-protocol,spec): a paged search's total is counted under the search engine.count() takes the query's search / searchFields and expands them through the one ADR-0061 expander find runs (expandSearchOnOptions, shared with aggregate); findData passes them to the count and the countable page-local estimate arm is gone. EngineCountOptionsSchema declares the pair exactly as EngineQueryOptionsSchema does, with EngineCountOptionsParsed. Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T Co-authored-by: Claude --- packages/metadata-protocol/src/protocol.ts | 33 ++++++-------- .../src/protocol.zero-limit-total.test.ts | 14 +++--- packages/objectql/src/engine.ts | 44 ++++++++++--------- packages/objectql/src/protocol-data.test.ts | 10 +++-- packages/spec/src/data/data-engine.test.ts | 14 ++++++ packages/spec/src/data/data-engine.zod.ts | 19 ++++++++ packages/spec/src/data/hook-api.test.ts | 8 ++-- packages/spec/src/data/hook-api.ts | 7 +-- .../src/type-alias-convention.pin.test.ts | 13 ++++-- 9 files changed, 102 insertions(+), 60 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 7c2cd7685a1..30b0a27b9a6 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -13292,10 +13292,11 @@ export class ObjectStackProtocolImplementation implements // whether more pages remain (true server-side pagination). Without a limit // the full result set is returned, so its length already IS the total. // - // engine.count() only honors `where`; a `search`/`distinct` query can't be - // reproduced by it, so for those we skip the count and fall back to a - // page-local estimate (a full page implies there may be more) rather than - // reporting a wrong total. + // [#22790] The count takes the query's `search` / `searchFields` too: the + // engine expands them through the one ADR-0061 expander `find` uses, so + // `total` counts exactly the rows this query pages. A search used to skip + // the count and report a page-local estimate (`offset + page + 1`) that + // the Console rendered as the record count. // // [#22588] `limit: 0` IS a limit. It asks for no rows (objectstack#6485, // the drivers' pagination conformance), so `records.length` is 0 and was @@ -13306,9 +13307,8 @@ export class ObjectStackProtocolImplementation implements // the full set is the total" arm, which is true of every limit except // this one. So `>= 0`: a zero-row request is a paged request like any // other, and the three arms below keep their meaning for it — the count - // runs (`total` is the filtered count, whatever the page size), `$count=false` - // still omits `total` instead of reporting the page's 0, and `search` - // still reports its page-local estimate rather than a count. + // runs (`total` is the filtered count, whatever the page size), and + // `$count=false` still omits `total` instead of reporting the page's 0. const pageLimit = typeof options.limit === 'number' && options.limit >= 0 ? options.limit : undefined; const pageOffset = typeof options.offset === 'number' && options.offset > 0 ? options.offset : 0; let total: number | undefined = records.length; @@ -13318,7 +13318,6 @@ export class ObjectStackProtocolImplementation implements // the flag's ONLY observable effect platform-wide, on a capability // that never deduplicated a row. Removed with `query.distinct` // (tombstoned in spec 17); `total`/`hasMore` are truthful again. - const countable = options.search == null; if (countOptOut) { // [#10757] The caller said it does not need `total`, so the // COUNT query is not issued at all — that is the whole point of @@ -13330,19 +13329,16 @@ export class ObjectStackProtocolImplementation implements // filter (IF REQUESTED)", // `packages/spec/src/api/protocol.zod.ts`), so absent is the // declared shape for "not requested" — and it is the only - // honest one: the `search` branch below reports an estimate - // because it has no better number to give, while here a real - // number was available and the caller declined it. Handing back - // a plausible-looking guess under those circumstances is how a + // honest one: a real number was available and the caller + // declined it. Handing back a plausible-looking guess is how a // page-local estimate ends up rendered as a record count. // // `hasMore` is still answered, from the page alone: a FULL page - // means there may be more. Same page-local rule the search - // branch uses, and it never over-reports the data — it can only - // say "maybe more" on an exactly-full last page. + // means there may be more. It never over-reports the data — it + // can only say "maybe more" on an exactly-full last page. hasMore = records.length === pageLimit; total = undefined; - } else if (countable) { + } else { // [#10757] `counted` is a separate, always-assigned local so // `hasMore` below compares against a `number`: `total` became // optional when `$count=false` gained the right to omit it, and @@ -13351,6 +13347,8 @@ export class ObjectStackProtocolImplementation implements try { counted = await this.engine.count(request.object, { where: options.where, + search: options.search, + searchFields: options.searchFields, context: options.context, } as any); } catch { @@ -13360,9 +13358,6 @@ export class ObjectStackProtocolImplementation implements } total = counted; hasMore = pageOffset + records.length < counted; - } else { - hasMore = records.length === pageLimit; - total = pageOffset + records.length + (hasMore ? 1 : 0); } } return { diff --git a/packages/metadata-protocol/src/protocol.zero-limit-total.test.ts b/packages/metadata-protocol/src/protocol.zero-limit-total.test.ts index af264104684..8d3d7853f80 100644 --- a/packages/metadata-protocol/src/protocol.zero-limit-total.test.ts +++ b/packages/metadata-protocol/src/protocol.zero-limit-total.test.ts @@ -30,9 +30,9 @@ * count for a zero-row page would pass (1) and fail here. * 3. The two neighbouring arms keep their meaning for a zero-row page: * `$count=false` still issues no COUNT and omits `total` (it no longer - * reports the page's `0` as if it were a count), and `search` still issues - * no COUNT — `engine.count()` cannot reproduce a search, which is why that - * arm reports a page-local estimate at every page size. + * reports the page's `0` as if it were a count), and a `search` is counted + * like any other filter (#22790: the COUNT carries the search the rows + * were read with). * 4. Absent a limit nothing moves: no COUNT, `total` is the set's length. */ @@ -149,10 +149,12 @@ describe('[#22588] the neighbouring arms keep their meaning for a zero-row page' expect('total' in result).toBe(false); }); - it('search still issues no COUNT — engine.count() cannot reproduce a search', async () => { - const { result, count } = await list({ $top: '0', $search: 't' }); + it('a search is counted, with the search the rows were read with (#22790)', async () => { + const { result, count, find } = await list({ $top: '0', $search: 't' }); - expect(count).not.toHaveBeenCalled(); + expect(count).toHaveBeenCalledTimes(1); + expect((count.mock.calls[0] as unknown[])[1]).toMatchObject({ search: 't' }); + expect((find.mock.calls[0] as unknown[])[1]).toMatchObject({ search: 't' }); expect(result.records).toEqual([]); }); diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 9905baf9fb8..4ff28f837aa 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -619,12 +619,13 @@ const ENGINE_DELETE_OPTION_KEYS: ReadonlySet = new Set([ 'context', 'where', 'multi', ...ENGINE_DRIVER_PASSTHROUGH_KEYS, ]); -const ENGINE_COUNT_OPTION_KEYS: ReadonlySet = new Set(['context', 'where']); -// `search` / `searchFields` are READ by `aggregate` through the same ADR-0061 -// expander `find` uses ({@link ObjectQL.expandSearchOnAggregateOptions}), so a -// grouped answer under a search is the grouping of the searched rows. Before -// `EngineAggregateOptionsSchema` declared them they were refused here, and the -// one wire path to this verb left them out — the unsearched groups answered. +// `search` / `searchFields` are READ by `count` and `aggregate` through the +// same ADR-0061 expander `find` uses ({@link ObjectQL.expandSearchOnOptions}), +// so a count or a grouped answer under a search is over the searched rows. +// Before their schemas declared them they were refused here, and the wire paths +// left them out: the unsearched groups answered, and a paged list's `total` +// under a search was a page-local estimate (#22790). +const ENGINE_COUNT_OPTION_KEYS: ReadonlySet = new Set(['context', 'where', 'search', 'searchFields']); const ENGINE_AGGREGATE_OPTION_KEYS: ReadonlySet = new Set([ 'context', 'where', 'groupBy', 'aggregations', 'having', 'timezone', 'search', 'searchFields', @@ -12679,13 +12680,15 @@ export class ObjectQL implements IObjectQLEngine { } /** - * The aggregate verb's door into {@link expandSearchOnAst} — ⛔ not a second - * expander. `QuerySchema.search` (ADR-0061 D1) is declared beside `groupBy` / - * `aggregations` with no carve-out, so a grouped read under a search groups - * the SEARCHED rows: the same cross-field `$or`, resolved from the same - * server-side searchable set and the same `searchFields` narrowing, ANDed - * with `where` exactly as on `find` — before the security middlewares run, - * so an injected RLS predicate composes with it the same way on both verbs. + * The aggregate and count verbs' door into {@link expandSearchOnAst} — ⛔ not + * a second expander. `QuerySchema.search` (ADR-0061 D1) is declared beside + * `groupBy` / `aggregations` with no carve-out, so a grouped read under a + * search groups the SEARCHED rows, and a count under one counts them (#22790: + * `findData`'s paged `total`): the same cross-field `$or`, resolved from the + * same server-side searchable set, `internal: true` exclusion and + * `searchFields` narrowing, ANDed with `where` exactly as on `find` — before + * the security middlewares run, so an injected RLS predicate composes with it + * the same way on every verb. * * The expansion runs on a carrier AST holding only the three keys it reads, * and the result is a COPY of `query` with `where` replaced and the two @@ -12694,16 +12697,13 @@ export class ObjectQL implements IObjectQLEngine { * `opCtx.options` unexpanded. The common path — no search key — returns the * same reference and allocates nothing. */ - private expandSearchOnAggregateOptions( - object: string, - query: EngineAggregateOptions, - ): EngineAggregateOptions { + private expandSearchOnOptions(object: string, query: T): T { if (query.search === undefined && query.searchFields === undefined) return query; const { search, searchFields, ...rest } = query; const carrier = { object, where: rest.where, search, searchFields } as QueryAST; this.expandSearchOnAst(carrier, this._registry.getObject(object)); - if (carrier.where === undefined) return rest; - return { ...rest, where: carrier.where }; + if (carrier.where === undefined) return rest as T; + return { ...rest, where: carrier.where } as T; } /** @@ -18695,6 +18695,10 @@ export class ObjectQL implements IObjectQLEngine { query = foldEngineOptionAliases(object, 'count', query, ENGINE_WHERE_SLOTS); rejectUnknownEngineOptions(object, 'count', query, ENGINE_COUNT_OPTION_KEYS); query = lowerWhereFilterArray(object, 'count', query, this._registry.getObject(object), this.relatedSchemaOf); + // [#22790] ADR-0061 `search` → the rows are searched before they are + // counted, by the one expander `find` runs, at the same point in the + // sequence — so the count is the number of rows `find` returns for the bag. + if (query) query = this.expandSearchOnOptions(object, query); const driver = this.getDriver(object); // The AST must ride on the opCtx so the security/sharing middlewares can @@ -18830,7 +18834,7 @@ export class ObjectQL implements IObjectQLEngine { // ADR-0061 `search` → the rows are searched BEFORE they are grouped, by // the one expander `find` runs, at the same point in the sequence (after // the `where` doors above, before the AST is built and tokens resolve). - query = this.expandSearchOnAggregateOptions(object, query); + query = this.expandSearchOnOptions(object, query); this.rejectCredentialAggregation(object, query); // [#20783] …and a `groupBy` entry naming a structured-JSON field (`json`, // `composite`, `address`, …) is refused `INVALID_FIELD` / 400 here, before diff --git a/packages/objectql/src/protocol-data.test.ts b/packages/objectql/src/protocol-data.test.ts index d8ee969496c..5c5de5606d2 100644 --- a/packages/objectql/src/protocol-data.test.ts +++ b/packages/objectql/src/protocol-data.test.ts @@ -287,14 +287,16 @@ describe('ObjectStackProtocolImplementation - Data Operations', () => { expect(result.hasMore).toBe(false); }); - it('skips count for search queries and estimates hasMore from a full page', async () => { - // engine.count() can't reproduce a $search, so we must not call it; a - // full page (length === limit) implies there may be more. + it('counts a search query under the same search, and reports that total (#22790)', async () => { + // The engine expands `search` through the one expander `find` uses, + // so the total is a count, never a page-local estimate. mockEngine.find.mockResolvedValue(Array.from({ length: 10 }, (_, i) => ({ id: `r${i}` }))); + mockEngine.count.mockResolvedValue(75); const result = await protocol.findData({ object: 'task', query: { $top: 10, $search: 'foo' } }); - expect(mockEngine.count).not.toHaveBeenCalled(); + expect(mockEngine.count).toHaveBeenCalledWith('task', expect.objectContaining({ search: 'foo' })); + expect(result.total).toBe(75); expect(result.hasMore).toBe(true); }); }); diff --git a/packages/spec/src/data/data-engine.test.ts b/packages/spec/src/data/data-engine.test.ts index 9906f701a92..c356f0c741e 100644 --- a/packages/spec/src/data/data-engine.test.ts +++ b/packages/spec/src/data/data-engine.test.ts @@ -598,6 +598,20 @@ describe('EngineCountOptionsSchema', () => { }); expect(options.where).toBeDefined(); }); + + // #22790 — a count under a search counts the searched rows (a paged list's + // `total`), so the count options declare the pair exactly as `find` does. + it('keeps `search` / `searchFields`, declared exactly as EngineQueryOptionsSchema declares them', () => { + const parsed = EngineCountOptionsSchema.parse({ search: { query: 'acme', fields: ['name'] }, searchFields: ['name'] }); + expect(parsed.search).toMatchObject({ query: 'acme', fields: ['name'] }); + expect(parsed.searchFields).toEqual(['name']); + const countShape = (EngineCountOptionsSchema as unknown as z.ZodObject).shape; + const findShape = (EngineQueryOptionsSchema as unknown as z.ZodObject).shape; + for (const key of ['search', 'searchFields'] as const) { + expect(z.toJSONSchema(countShape[key] as z.ZodType, { io: 'input' }), key) + .toEqual(z.toJSONSchema(findShape[key] as z.ZodType, { io: 'input' })); + } + }); }); describe('DataEngineFindRequestSchema', () => { diff --git a/packages/spec/src/data/data-engine.zod.ts b/packages/spec/src/data/data-engine.zod.ts index cc438fb83fd..d63e4810634 100644 --- a/packages/spec/src/data/data-engine.zod.ts +++ b/packages/spec/src/data/data-engine.zod.ts @@ -483,6 +483,20 @@ export const DataEngineAggregateOptionsSchema = lazySchema(() => BaseEngineOptio export const EngineCountOptionsSchema = lazySchema(() => BaseEngineOptionsSchema.extend({ /** Filter conditions — standard QueryAST `where` */ where: z.union([z.record(z.string(), z.unknown()), FilterConditionSchema]).optional(), + /** + * Full-Text Search over the rows BEFORE they are counted — declared exactly + * as {@link EngineQueryOptionsSchema}'s `search` (ADR-0061 D1). The engine + * expands it through the same `$search` → cross-field `$or` expansion `find` + * runs and ANDs it with `where`, so a count under a search is the number of + * rows `find` returns for the same bag — the `total` a paged list reports. + */ + search: z.union([z.string(), FullTextSearchSchema]).optional(), + /** + * Fields the `search` expansion may match against — declared exactly as + * {@link EngineQueryOptionsSchema}'s `searchFields`: intersected with the + * object's declared/derived searchable set (ADR-0061), never widened. + */ + searchFields: z.array(z.string()).optional(), }).describe('QueryAST-aligned options for DataEngine.count operations')); // -------------------------------------------------------------------------- @@ -1468,6 +1482,11 @@ export type EngineAggregateOptions = z.input; export type EngineCountOptions = z.input; +/** + * Post-parse shape of {@link EngineCountOptions} — defaults applied (ADR-0122). + * The two shapes part at `search`, exactly as on {@link EngineQueryOptionsParsed}. + */ +export type EngineCountOptionsParsed = z.infer; // --- Legacy: deprecated types (kept for backward compatibility) --- export type DataEngineFilter = z.input; diff --git a/packages/spec/src/data/hook-api.test.ts b/packages/spec/src/data/hook-api.test.ts index bd9db9af2f6..139fe1eb857 100644 --- a/packages/spec/src/data/hook-api.test.ts +++ b/packages/spec/src/data/hook-api.test.ts @@ -134,11 +134,11 @@ describe('HookApi — the published hook ctx.api face', () => { }); describe('count is not find narrowed by habit', () => { - it('takes `where` and refuses everything else', () => { - const ok: HookCountQuery = { where: { status: 'active' } }; + it('takes `where` and the search pair, and refuses everything else', () => { + const ok: HookCountQuery = { where: { status: 'active' }, search: 'acme', searchFields: ['name'] }; const withLimit: HookCountQuery = { // @ts-expect-error `count` honours no pagination — `ENGINE_COUNT_OPTION_KEYS` - // is `{ context, where }` and the engine rejects anything else. + // is `{ context, where, search, searchFields }` and the engine rejects anything else. limit: 5, }; const withPassthrough: HookCountQuery = { @@ -210,7 +210,7 @@ describe('HookApi — the published hook ctx.api face', () => { { name: 'HookCountQuery / EngineCountOptionsSchema', schema: EngineCountOptionsSchema, - kept: ['where'], + kept: ['where', 'search', 'searchFields'], omitted: ['context'], }, { diff --git a/packages/spec/src/data/hook-api.ts b/packages/spec/src/data/hook-api.ts index 18268f41022..8fa10d0072b 100644 --- a/packages/spec/src/data/hook-api.ts +++ b/packages/spec/src/data/hook-api.ts @@ -113,7 +113,7 @@ import type { * than re-spelled so there is one declaration of each key's type. * * ⚠️ They are legal on `find` / `findOne` / `update` / `delete` and NOT on - * `count`, which forwards no bag at all and whose legal set is `where` alone. + * `count`, which forwards no bag at all and whose legal set is `where` and the search pair. * That asymmetry is engine behaviour, it is invisible from any document, and it * is precisely the kind of thing a hand-written copy gets wrong — which is why * {@link HookCountQuery} below is the one shape that does not carry these. @@ -144,11 +144,12 @@ export type HookQuery = Omit; diff --git a/packages/spec/src/type-alias-convention.pin.test.ts b/packages/spec/src/type-alias-convention.pin.test.ts index e677bf0cdf4..ba70d00d470 100644 --- a/packages/spec/src/type-alias-convention.pin.test.ts +++ b/packages/spec/src/type-alias-convention.pin.test.ts @@ -274,7 +274,7 @@ import type * as M187 from './shared/duration.zod.js'; import type * as M188 from './ai/build-progress.zod.js'; // --------------------------------------------------------------------------- -// 776 isomorphic aliases: `z.input` === `z.infer`, so no `XParsed` is declared. +// 775 isomorphic aliases: `z.input` === `z.infer`, so no `XParsed` is declared. // // That number is machine-checked, not hand-kept. The runtime companion at the // bottom of this file recomputes the pin count from the source and asserts that @@ -786,7 +786,6 @@ export type Iso_data_dataEngine__DataEngineInsertRequestSchema = Assert, z.infer< typeof M56.DataEngineUpdateOptionsSchema > >>; export type Iso_data_dataEngine__DataEngineVectorFindRequestSchema = Assert, z.infer< typeof M56.DataEngineVectorFindRequestSchema > >>; export type Iso_data_dataEngine__DroppedFieldsEventSchema = Assert, z.infer< typeof M56.DroppedFieldsEventSchema > >>; -export type Iso_data_dataEngine__EngineCountOptionsSchema = Assert, z.infer< typeof M56.EngineCountOptionsSchema > >>; export type Iso_data_dataEngine__EngineDeleteOptionsSchema = Assert, z.infer< typeof M56.EngineDeleteOptionsSchema > >>; export type Iso_data_dataEngine__EngineUpdateOptionsSchema = Assert, z.infer< typeof M56.EngineUpdateOptionsSchema > >>; export type Iso_data_dataEngine__ValidationAdvisoryEventSchema = Assert, z.infer< typeof M56.ValidationAdvisoryEventSchema > >>; @@ -1674,7 +1673,7 @@ describe('ADR-0122 type-alias convention', () => { // this title and the section header above the pin list — are now asserted // against the recomputed count below, so neither can go stale without a red // test naming it. - it('still declares all 776 isomorphic pins', () => { + it('still declares all 775 isomorphic pins', () => { // The truth of each pin is proved by tsc, not here — an `Assert>` // that stops holding is a compile error with the alias named. What tsc // cannot notice is a pin that was DELETED: removing the assertion removes @@ -2455,7 +2454,13 @@ describe('ADR-0122 type-alias convention', () => { // tree, so it is pinned here // (Iso_data_dataEngine__ValidationAdvisoryEventSchema) rather than given a // `ValidationAdvisoryEventParsed` synonym. +1 added. - expect(pins).toHaveLength(776); + // + // 776 -> 775 is #22790, the count verb honouring ADR-0061 `search`: + // `EngineCountOptionsSchema` gained `search` / `searchFields`, declared + // exactly as on `EngineQueryOptionsSchema`, so input !== infer as on the + // aggregate verb above. Iso_data_dataEngine__EngineCountOptionsSchema + // leaves and `EngineCountOptionsParsed` is declared. -1 removed. + expect(pins).toHaveLength(775); // The count is stated in PROSE twice as well — this case's title and the // section header above the pin list — and until commit c6b05c76a nothing read either From e85d2a4d1324456bed64c2ac9301a4efc396e4f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 12:15:57 +0000 Subject: [PATCH 3/4] chore(spec): regenerate artifacts for EngineCountOptions search pair Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T Co-authored-by: Claude --- content/docs/references/data/data-engine.mdx | 23 ++++++++++++++++++-- packages/spec/api-surface/data.json | 1 + packages/spec/authorable-surface/data.json | 2 ++ packages/spec/export-origins/data.json | 1 + 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/content/docs/references/data/data-engine.mdx b/content/docs/references/data/data-engine.mdx index ce6fd0057e0..abf693a3371 100644 --- a/content/docs/references/data/data-engine.mdx +++ b/content/docs/references/data/data-engine.mdx @@ -221,7 +221,7 @@ Options for DataEngine.count operations | :--- | :--- | :--- | :--- | | **method** | `'count'` | ✅ | | | **object** | `string` | ✅ | | -| **query** | `{ context?: object; where?: Record \| any; filter?: Record \| any }` | optional | | +| **query** | `{ context?: object; where?: Record \| any; search?: string \| object; searchFields?: string[]; … }` | optional | | ### Nested Shape: `DataEngineCountRequest.query` @@ -229,6 +229,8 @@ Options for DataEngine.count operations | :--- | :--- | :--- | :--- | | **context** | `{ userId?: string; actor?: string; attributedUserId?: string; email?: string; … }` | optional | | | **where** | `Record \| any` | optional | | +| **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | | +| **searchFields** | `string[]` | optional | | | **filter** | `Record \| any` | optional | Data Engine query filter conditions | @@ -684,7 +686,7 @@ This schema accepts one of the following structures: | :--- | :--- | :--- | :--- | | **method** | `'count'` | ✅ | | | **object** | `string` | ✅ | | -| **query** | `{ context?: object; where?: Record \| any; filter?: Record \| any }` | optional | | +| **query** | `{ context?: object; where?: Record \| any; search?: string \| object; searchFields?: string[]; … }` | optional | | ### Nested Shape: `DataEngineRequest[method='count'].query` @@ -692,6 +694,8 @@ This schema accepts one of the following structures: | :--- | :--- | :--- | :--- | | **context** | `{ userId?: string; actor?: string; attributedUserId?: string; email?: string; … }` | optional | | | **where** | `Record \| any` | optional | | +| **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | | +| **searchFields** | `string[]` | optional | | | **filter** | `Record \| any` | optional | Data Engine query filter conditions | --- @@ -994,6 +998,8 @@ QueryAST-aligned options for DataEngine.count operations | :--- | :--- | :--- | :--- | | **context** | `{ userId?: string; actor?: string; attributedUserId?: string; email?: string; … }` | optional | | | **where** | `Record \| any` | optional | | +| **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | | +| **searchFields** | `string[]` | optional | | ### Nested Shape: `EngineCountOptions.context` @@ -1033,6 +1039,19 @@ QueryAST-aligned options for DataEngine.count operations | **transaction** | `any` | optional | | | **traceId** | `string` | optional | | +### Nested Shape: `EngineCountOptions.search` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **query** | `string` | ✅ | Search query text | +| **fields** | `string[]` | optional | Fields to search in (if not specified, searches all text fields) | +| **fuzzy** | `boolean` | optional (default: `false`) | [EXPERIMENTAL — not enforced] Fuzzy matching (tolerate typos). The ADR-0061 expansion reads only `query` + `fields`; no executor receives this flag. | +| **operator** | `Enum<'and' \| 'or'>` | optional (default: `"or"`) | [EXPERIMENTAL — not enforced] Logical operator between terms. The ADR-0061 expansion applies its own term semantics; no executor receives this flag. | +| **boost** | `Record` | optional | [EXPERIMENTAL — not enforced] Field-specific relevance boosting (field name -> boost factor). No executor scores results. | +| **minScore** | `number` | optional | [EXPERIMENTAL — not enforced] Minimum relevance score threshold. No executor scores results. | +| **language** | `string` | optional | [EXPERIMENTAL — not enforced] Language for text analysis (e.g., "en", "zh", "es"). No executor selects an analyzer. | +| **highlight** | `boolean` | optional (default: `false`) | [EXPERIMENTAL — not enforced] Search result highlighting. No executor emits highlights. | + --- diff --git a/packages/spec/api-surface/data.json b/packages/spec/api-surface/data.json index 67cd1b8c87d..b0efb6ccda0 100644 --- a/packages/spec/api-surface/data.json +++ b/packages/spec/api-surface/data.json @@ -255,6 +255,7 @@ "EngineAggregateOptionsParsed (type)", "EngineAggregateOptionsSchema (const)", "EngineCountOptions (type)", + "EngineCountOptionsParsed (type)", "EngineCountOptionsSchema (const)", "EngineDeleteOptions (type)", "EngineDeleteOptionsSchema (const)", diff --git a/packages/spec/authorable-surface/data.json b/packages/spec/authorable-surface/data.json index 2ff4eaa6dcc..c3a152ecaa0 100644 --- a/packages/spec/authorable-surface/data.json +++ b/packages/spec/authorable-surface/data.json @@ -268,6 +268,8 @@ "data/EngineAggregateOptions:timezone", "data/EngineAggregateOptions:where", "data/EngineCountOptions:context", + "data/EngineCountOptions:search", + "data/EngineCountOptions:searchFields", "data/EngineCountOptions:where", "data/EngineDeleteOptions:context", "data/EngineDeleteOptions:multi", diff --git a/packages/spec/export-origins/data.json b/packages/spec/export-origins/data.json index 5ae0ef22d00..b0add27ab08 100644 --- a/packages/spec/export-origins/data.json +++ b/packages/spec/export-origins/data.json @@ -250,6 +250,7 @@ "EngineAggregateOptionsParsed": "src/data/data-engine.zod.ts#EngineAggregateOptionsParsed (type)", "EngineAggregateOptionsSchema": "src/data/data-engine.zod.ts#EngineAggregateOptionsSchema (const)", "EngineCountOptions": "src/data/data-engine.zod.ts#EngineCountOptions (type)", + "EngineCountOptionsParsed": "src/data/data-engine.zod.ts#EngineCountOptionsParsed (type)", "EngineCountOptionsSchema": "src/data/data-engine.zod.ts#EngineCountOptionsSchema (const)", "EngineDeleteOptions": "src/data/data-engine.zod.ts#EngineDeleteOptions (type)", "EngineDeleteOptionsSchema": "src/data/data-engine.zod.ts#EngineDeleteOptionsSchema (const)", From 4e057d908e16aae99375c697f7420e789ffc5344 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 12:38:14 +0000 Subject: [PATCH 4/4] chore: changeset for the counted search total; soft page assertions in the door pin Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T Co-authored-by: Claude --- .changeset/22790-search-total-counted.md | 17 +++++++++++++++++ .../rest/src/list-search-total-door.test.ts | 9 +++++---- 2 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 .changeset/22790-search-total-counted.md diff --git a/.changeset/22790-search-total-counted.md b/.changeset/22790-search-total-counted.md new file mode 100644 index 00000000000..6f92b8155a2 --- /dev/null +++ b/.changeset/22790-search-total-counted.md @@ -0,0 +1,17 @@ +--- +"@objectstack/spec": minor +"@objectstack/objectql": minor +"@objectstack/metadata-protocol": patch +--- + +A paged list under a search now reports the real number of matching records as `total`, and `hasMore` agrees with it on every page. + +Clause-②: yes (widening) — `EngineCountOptionsSchema` gains two OPTIONAL keys, `search` and `searchFields`, so the accept set of the count options grows. Nothing previously admitted is refused, no key is renamed or retired, and no producer is required to write them. + +Until now `GET /api/v1/data/:object?$top=25&$search=Line` over 75 matching records answered `total: 26` (offset + page + 1, a page-local estimate), and the last page claimed `hasMore: true`; `POST /api/v1/data/:object/query` with a `search` did the same. Nothing on the response marked the number as an estimate, so the Console rendered it as the record count and offered the wrong number of pages. + +- **`@objectstack/spec`** — `EngineCountOptionsSchema` declares `search` (the bare string, or the structured `FullTextSearchSchema` form) and `searchFields`, identically to `EngineQueryOptionsSchema`. FROM `count(object, { where })` only → TO `count(object, { where, search, searchFields })`. The structured `search` form carries flag defaults, so the parsed state is named `EngineCountOptionsParsed` (ADR-0122). `HookCountQuery` (`ctx.api.object(name).count()`) derives from it and gains the same two keys. +- **`@objectstack/objectql`** — `engine.count()` (and `ctx.api.object(name).count()`) accepts the two keys it used to refuse as unknown options, and expands them through the same ADR-0061 expansion `find()` and `aggregate()` use: the same server-resolved searchable fields, the same `internal: true` exclusion, the same `searchFields` narrowing, AND-ed with `where` before the security middlewares run. There is one expander, not two, so a count under a search is the number of rows `find()` returns for the same bag. +- **`@objectstack/metadata-protocol`** — `findData`'s paged branch passes `search` / `searchFields` to `engine.count()`, and the page-local estimate branch is gone. `$count=false` still skips the count and omits `total`. + +Nothing to migrate. A client that worked around the estimate, for example by labelling the count as page-local under a search, can read `total` as the match count again. diff --git a/packages/rest/src/list-search-total-door.test.ts b/packages/rest/src/list-search-total-door.test.ts index a00b98e0958..12d5482deba 100644 --- a/packages/rest/src/list-search-total-door.test.ts +++ b/packages/rest/src/list-search-total-door.test.ts @@ -115,13 +115,14 @@ describe('[#22790] a paged search reports the counted total', () => { const unpaged = await list({ $search: 'Line' }); expect(unpaged.records).toHaveLength(75); + // Soft, so a red run names every wrong number — the first page AND the last. const first = await list({ $top: '25', $search: 'Line' }); - expect(first).toMatchObject({ total: 75, hasMore: true }); - expect(first.records).toHaveLength(25); + expect.soft(first).toMatchObject({ total: 75, hasMore: true }); + expect.soft(first.records).toHaveLength(25); const last = await list({ $top: '25', $skip: '50', $search: 'Line' }); - expect(last).toMatchObject({ total: 75, hasMore: false }); - expect(last.records).toHaveLength(25); + expect.soft(last).toMatchObject({ total: 75, hasMore: false }); + expect.soft(last.records).toHaveLength(25); }); it('POST …/query: the same, and a `searchFields` narrowing narrows the count with the rows', async () => {