From 44fc3f645839c3a5a5eef3463826dcc0098bf089 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 18:15:47 +0000 Subject: [PATCH 1/4] feat(core): type a collection's content filters and sorts from its content type filter.content, filter.contentPresent and sort.contentField were untyped inside a collection, so a typo'd field or enum value compiled and silently matched nothing. They are now derived from the content type C: - ContentPathOf: top-level fields plus dotted paths through declared objects and arrays, any suffix below an open node, typed six segments deep to stay within the compiler's instantiation limit. - SortableFieldOf: top-level scalars only. Matching still runs on content as stored; the spec says so and points at migrateAll() for records at an older version. Closes #410 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KQtAGzBB8fDpfqZhc84ZdN --- .changeset/typed-collection-filters.md | 5 + docs/spec/data-model.md | 2 +- packages/core/src/collection.ts | 75 +++++++++++++- packages/core/src/index.ts | 2 + packages/core/tests/collection.test.ts | 137 ++++++++++++++++++++++++- 5 files changed, 215 insertions(+), 6 deletions(-) create mode 100644 .changeset/typed-collection-filters.md diff --git a/.changeset/typed-collection-filters.md b/.changeset/typed-collection-filters.md new file mode 100644 index 00000000..d59ad8ad --- /dev/null +++ b/.changeset/typed-collection-filters.md @@ -0,0 +1,5 @@ +--- +'@haverstack/core': minor +--- + +A collection's `filter.content`, `filter.contentPresent` and `sort.contentField` are typed from its content type: a typo'd field, a value outside an enum or a nested sort field no longer compiles. Adds `ContentPathOf` and `SortableFieldOf`. diff --git a/docs/spec/data-model.md b/docs/spec/data-model.md index f47dbac8..78c1c8e7 100644 --- a/docs/spec/data-model.md +++ b/docs/spec/data-model.md @@ -426,7 +426,7 @@ A record at an older version that no passed migration carries to the handle's is **A subscription covers the family.** A change's `record`, when present, is migrated to the handle's version; the change's own `typeId` stays the stored one, as `RecordChange.typeId` documents. A change whose record has `deletedAt` set carries neither `record` nor `misfit`, on either surface, and neither does a change that [migrates the record out](#type-migrations) of the family, which the filter matches by its previous `typeId`. The change filter takes every key but `typeId` and `baseId`. -**Content filters and sorts apply to content as stored.** The adapter runs them before any record is migrated, so a field a migration renamed or derived is matched and ordered under its stored name: filtering an `@2` collection on a field `@1` spelled differently does not find `@1` records. `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. +**Content filters and sorts apply to content as stored.** The adapter runs them before any record is migrated, so a field a migration renamed or derived is matched and ordered under its stored name: filtering an `@2` collection on a field `@1` spelled differently does not find `@1` records. Their keys are typed against the handle's version: `filter.content` and `filter.contentPresent` take the content's paths (`ContentPathOf`), through declared objects and arrays, with any suffix below an `open` object or array, and a value of the field's type (an array's element type, since matching is element-wise) or `null`; `sort.contentField` takes only a top-level scalar (`SortableFieldOf`). The compiler therefore accepts a key that records still at an older version may spell differently, and only [`migrateAll()`](#type-migrations) makes the stored shape match the one the keys were checked against. Path typing stops six segments deep and accepts any suffix below that, a limit of the compiler, not of the [32-segment path](#nested-content-paths). `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. **Every verb that returns a record returns it as `get()` would.** The write already returned the record, so narrowing it costs no round trip, and a `restoreVersion()` to a pre-migration snapshot reads back migrated, like any older record. A write is checked against the read it already makes, before anything lands: a record outside the family is refused with `StackNotFoundError`, the answer `get()` gives, and so is a `restoreVersion()` to a snapshot taken before the record migrated into the family, which is not a version of a record in the collection. A content write to a record stored at a newer version is refused with `MisfitError` before anything is written, since its patch would be validated against a schema the handle does not know. Any other write that leaves a misfit lands, then throws `MisfitError` carrying the stored record, so the app knows the write landed. A deleted record, or a stale `ifVersion`, is refused first, as the untyped write refuses it. diff --git a/packages/core/src/collection.ts b/packages/core/src/collection.ts index 9ddf9676..f5da81ea 100644 --- a/packages/core/src/collection.ts +++ b/packages/core/src/collection.ts @@ -33,6 +33,7 @@ import type { ChangeFilter, DataAssociation, IfVersionOptions, + NativeSortField, RecordChange, RecordChangeSet, RecordFilter, @@ -97,14 +98,80 @@ export type CollectionPage = { cursor: string | null; }; +/** + * Path typing stops at this depth and accepts any suffix below it, since a + * recursive path type otherwise exceeds the compiler's instantiation limit. + * A typing limit, not a query limit: see docs/spec/data-model.md § Nested + * content paths. + */ +type PathDepth = [never, 0, 1, 2, 3, 4, 5]; + +/** + * Each content path of `V` reached at `P`, paired with the value a filter + * compares there. An array is matched element-wise, so it is its element. + */ +type PathEntry

= [D] extends [never] + ? [P | `${P}.${string}`, unknown] + : V extends readonly (infer E)[] + ? unknown extends E + ? [P | `${P}.${string}`, unknown] + : PathEntry + : V extends object + ? string extends keyof V + ? [P, V] | [`${P}.${string}`, unknown] + : [P, V] | FieldEntry + : [P, V]; + +/** `Required[K]` rather than `NonNullable`, which would leak into errors. */ +type FieldEntry = [D] extends [never] + ? [`${Prefix}${string}`, unknown] + : { + [K in keyof C & string]-?: PathEntry<`${Prefix}${K}`, Required[K], D>; + }[keyof C & string]; + +/** + * Every content path of `C`: its fields, and dotted paths through declared + * objects and arrays. Below an `open` object or array, or six segments + * deep, any suffix is a path. + */ +export type ContentPathOf = FieldEntry[0]; + +/** The top-level scalar fields of `C`, the ones a content sort reaches. */ +export type SortableFieldOf = string extends keyof C + ? string + : { + [K in keyof C & string]-?: Required[K] extends string | number | boolean ? K : never; + }[keyof C & string]; + /** * A collection reads live records of its own family only, as stored, so * none of `typeId`, `baseId`, `includeDeleted` or `presentAt` is offered. - * `C` is reserved for typing content filters and sorts. + * Content filter keys and the sort field are checked against `C`, the + * handle's version, while matching runs on content as stored. See + * docs/spec/data-model.md § Type handles. */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -export type CollectionQuery = Omit & { - filter?: Omit; +export type CollectionQuery = Omit & { + filter?: Omit< + RecordFilter, + 'typeId' | 'baseId' | 'includeDeleted' | 'content' | 'contentPresent' + > & { + content?: { [E in FieldEntry as E[0]]?: E[1] | null }; + contentPresent?: FieldEntry[0][]; + }; + sort?: + | { field: NativeSortField; contentField?: never; direction?: 'asc' | 'desc' } + | { + field?: never; + // Spelled out rather than SortableFieldOf, so an error lists the fields. + contentField: string extends keyof C + ? string + : { + [K in keyof C & string]-?: Required[K] extends string | number | boolean + ? K + : never; + }[keyof C & string]; + direction?: 'asc' | 'desc'; + }; }; /** The family is the collection's, so neither `typeId` nor `baseId` is a filter key. */ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 52a0963f..df53481f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -61,7 +61,9 @@ export type { CollectionChangeSet, CollectionSubscribeOptions, ContentPatch, + ContentPathOf, Misfit, + SortableFieldOf, StoredContent, } from './collection.js'; diff --git a/packages/core/tests/collection.test.ts b/packages/core/tests/collection.test.ts index 62ae42e7..7e3d188e 100644 --- a/packages/core/tests/collection.test.ts +++ b/packages/core/tests/collection.test.ts @@ -2,14 +2,17 @@ import { describe, test, expect, expectTypeOf, beforeEach, vi } from 'vitest'; import { Stack } from '../src/stack/stack.js'; import { MemoryAdapter } from '../src/testing.js'; import { migration, typeHandle } from '../src/type-handle.js'; -import type { ContentOf, UnlistedValue } from '../src/type-handle.js'; +import type { ContentOf, ReadonlyTypeSchema, UnlistedValue } from '../src/type-handle.js'; import { MisfitError } from '../src/collection.js'; import type { Collection, CollectionChange, CollectionPage, + CollectionQuery, CollectionRecord, ContentPatch, + ContentPathOf, + SortableFieldOf, StoredContent, } from '../src/collection.js'; import { @@ -580,3 +583,135 @@ describe('a collection under delegation', () => { expect((await books.patchContent(old.id, { pages: 412 })).content.pages).toBe(412); }); }); + +describe('content filter and sort typing', () => { + const Contact = typeHandle({ + id: 'com.example.people/contact@1', + name: 'Contact', + schema: { + name: { kind: 'string', required: true }, + status: { kind: 'enum', values: ['active', 'archived'], required: true }, + age: { kind: 'number' }, + address: { + kind: 'object', + properties: { + city: { kind: 'string' }, + geo: { kind: 'object', properties: { lat: { kind: 'number' } } }, + }, + }, + emails: { + kind: 'array', + items: { kind: 'object', properties: { value: { kind: 'string', required: true } } }, + }, + tags: { kind: 'array', items: { kind: 'string' } }, + extra: { kind: 'object', open: true }, + bag: { kind: 'array', open: true }, + }, + }); + type Q = CollectionQuery>; + + test('derives paths and sortable fields from the content type', () => { + type C = ContentOf; + expectTypeOf>().toEqualTypeOf<'name' | 'status' | 'age'>(); + expectTypeOf>().toEqualTypeOf< + | 'name' + | 'status' + | 'age' + | 'address' + | 'address.city' + | 'address.geo' + | 'address.geo.lat' + | 'emails' + | 'emails.value' + | 'tags' + | 'extra' + | `extra.${string}` + | 'bag' + | `bag.${string}` + >(); + }); + + test('accepts declared paths, element values, any suffix below an open node, and null', async () => { + const stack = await Stack.open( + await MemoryAdapter.open({ ownerEntityId: OWNER, timezone: 'UTC' }), + ); + await stack.defineType(Contact); + const contacts = stack.collection(Contact); + const ada = await contacts.create({ + name: 'Ada', + status: 'active', + address: { city: 'London', geo: { lat: 51.5 } }, + emails: [{ value: 'ada@example.com' }], + tags: ['starred'], + extra: { source: { kind: 'import' } }, + bag: [1], + }); + await contacts.create({ name: 'Bea', status: 'archived', age: 40 }); + const { records } = await contacts.query({ + filter: { + content: { + status: 'active', + 'address.city': 'London', + 'address.geo.lat': 51.5, + 'emails.value': 'ada@example.com', + tags: 'starred', + 'extra.source.kind': 'import', + bag: 1, + age: null, + }, + contentPresent: ['address.city', 'extra.source'], + }, + sort: { contentField: 'name', direction: 'desc' }, + }); + expect(records.map((r) => r.id)).toEqual([ada.id]); + }); + + test('refuses a typo, a value outside an enum, a wrong nested type and a path through a scalar', () => { + // @ts-expect-error a typo'd field + const typo: Q = { filter: { content: { stauts: 'active' } } }; + // @ts-expect-error a value outside the enum + const badEnum: Q = { filter: { content: { status: 'actvie' } } }; + // @ts-expect-error a wrong value type at a nested path + const badNested: Q = { filter: { content: { 'address.geo.lat': 'north' } } }; + // @ts-expect-error a path through a scalar + const throughScalar: Q = { filter: { content: { 'name.first': 'Ada' } } }; + // @ts-expect-error a typo'd presence path + const badPresent: Q = { filter: { contentPresent: ['adress.city'] } }; + expect([typo, badEnum, badNested, throughScalar, badPresent]).toHaveLength(5); + }); + + test('sorts by top-level scalars only', () => { + // @ts-expect-error a nested field + const nested: Q = { sort: { contentField: 'address.city' } }; + // @ts-expect-error an object field + const object: Q = { sort: { contentField: 'address' } }; + // @ts-expect-error a typo'd field + const typo: Q = { sort: { contentField: 'nmae' } }; + expect([nested, object, typo]).toHaveLength(3); + }); + + test('types paths deeper than the typing limit without exceeding the compiler', () => { + type Obj

= { kind: 'object'; properties: P }; + type DeepQ = CollectionQuery< + ContentOf<{ + a: Obj<{ + b: Obj<{ + c: Obj<{ d: Obj<{ e: Obj<{ f: Obj<{ g: Obj<{ h: { kind: 'number' } }> }> }> }> }>; + }>; + }>; + }> + >; + const ok: DeepQ = { filter: { content: { 'a.b.c.d.e': {}, 'a.b.c.d.e.f.g.anything': 1 } } }; + // @ts-expect-error a typo above the limit is still caught + const typo: DeepQ = { filter: { content: { 'a.x': 1 } } }; + expect([ok, typo]).toHaveLength(2); + }); + + test('an untyped handle takes any path and sort field', () => { + const ok: CollectionQuery> = { + filter: { content: { anything: 1, 'a.b': 'x' }, contentPresent: ['q.r'] }, + sort: { contentField: 'x' }, + }; + expect(ok).toBeDefined(); + }); +}); From 846775fa1a54acbcc668aed47fc0378418b6d666 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 19:50:42 +0000 Subject: [PATCH 2/4] feat(core): narrow a collection's content filters and sorts to the handle's version A content filter or sort runs on content as stored, and a migration is an opaque function, so a predicate typed in the handle's shape cannot be translated into an older version's: run across the family, it silently missed or misplaced records whose field a migration renamed or remapped. A collection query with a content filter or sort now reads only records stored at the handle's version, and its page sets skippedOlder when older records match the rest of the query. Declarative migrations that would let predicates span versions are filed as #448. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KQtAGzBB8fDpfqZhc84ZdN --- .changeset/typed-collection-filters.md | 2 + docs/spec/data-model.md | 4 +- packages/core/src/collection.ts | 53 ++++++++++++++++++++++++-- packages/core/tests/collection.test.ts | 36 +++++++++++++++++ 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/.changeset/typed-collection-filters.md b/.changeset/typed-collection-filters.md index d59ad8ad..f0db7c6f 100644 --- a/.changeset/typed-collection-filters.md +++ b/.changeset/typed-collection-filters.md @@ -3,3 +3,5 @@ --- A collection's `filter.content`, `filter.contentPresent` and `sort.contentField` are typed from its content type: a typo'd field, a value outside an enum or a nested sort field no longer compiles. Adds `ContentPathOf` and `SortableFieldOf`. + +A collection query with a content filter or sort reaches only records stored at the handle's version, since a predicate in the handle's shape cannot be translated into an older version's. Its page sets `skippedOlder` when records at an older version match the rest of the query. diff --git a/docs/spec/data-model.md b/docs/spec/data-model.md index 78c1c8e7..0336dc1a 100644 --- a/docs/spec/data-model.md +++ b/docs/spec/data-model.md @@ -418,7 +418,7 @@ A read types an `enum` field as its listed values or any other string (`'want' | **A misfit is a record of the family the handle cannot represent**: one stored at a version newer than the handle's (`reason: 'newer-version'`), which cannot be migrated down. A misfit is reported, never thrown for a page: -- `query()` returns `{ records, misfits, cursor }`. Misfits do not count as records, so a page can hold fewer than `limit` records with a cursor still set: `cursor === null` ends the results, a short page does not. The page and its cursor survive one record the app cannot read. +- `query()` returns `{ records, misfits, cursor, skippedOlder }`. Misfits do not count as records, so a page can hold fewer than `limit` records with a cursor still set: `cursor === null` ends the results, a short page does not. The page and its cursor survive one record the app cannot read. - `get()` is a read the app asked for by ID, so it throws `MisfitError`, carrying the `reason` and the record as stored. - `subscribe()` delivers a change whose record does not fit with `misfit` in place of `record`. @@ -426,7 +426,7 @@ A record at an older version that no passed migration carries to the handle's is **A subscription covers the family.** A change's `record`, when present, is migrated to the handle's version; the change's own `typeId` stays the stored one, as `RecordChange.typeId` documents. A change whose record has `deletedAt` set carries neither `record` nor `misfit`, on either surface, and neither does a change that [migrates the record out](#type-migrations) of the family, which the filter matches by its previous `typeId`. The change filter takes every key but `typeId` and `baseId`. -**Content filters and sorts apply to content as stored.** The adapter runs them before any record is migrated, so a field a migration renamed or derived is matched and ordered under its stored name: filtering an `@2` collection on a field `@1` spelled differently does not find `@1` records. Their keys are typed against the handle's version: `filter.content` and `filter.contentPresent` take the content's paths (`ContentPathOf`), through declared objects and arrays, with any suffix below an `open` object or array, and a value of the field's type (an array's element type, since matching is element-wise) or `null`; `sort.contentField` takes only a top-level scalar (`SortableFieldOf`). The compiler therefore accepts a key that records still at an older version may spell differently, and only [`migrateAll()`](#type-migrations) makes the stored shape match the one the keys were checked against. Path typing stops six segments deep and accepts any suffix below that, a limit of the compiler, not of the [32-segment path](#nested-content-paths). `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. +**A content filter or sort reaches only records stored at the handle's version.** The adapter matches and orders content as stored, before any record is migrated, and a migration is an opaque function, so a predicate written in the handle's shape cannot be translated into an older version's. Run across the family, a field a migration renamed, derived or remapped would silently miss the older records or misplace them. So a `query()` whose `filter.content` or `filter.contentPresent` is non-empty, or whose sort is a `contentField`, reads only records whose `typeId` is the handle's, and its page sets `skippedOlder` when records at an older version match the rest of the query: the page may then be missing records the filter was meant to find. Mixed versions are a lasting state, not a backlog, since an app at an older version may keep writing its own, so the app decides what `skippedOlder` means to it; `migrateAll()` clears it only until the next such write. Such a page carries no misfits, since whether a newer record matches is equally unknowable. A query with no content keys reads the whole family. The keys are typed against the handle's version: `filter.content` and `filter.contentPresent` take the content's paths (`ContentPathOf`), through declared objects and arrays, with any suffix below an `open` object or array, and a value of the field's type (an array's element type, since matching is element-wise) or `null`; `sort.contentField` takes only a top-level scalar (`SortableFieldOf`). Path typing stops six segments deep and accepts any suffix below that, a limit of the compiler, not of the [32-segment path](#nested-content-paths). `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. **Every verb that returns a record returns it as `get()` would.** The write already returned the record, so narrowing it costs no round trip, and a `restoreVersion()` to a pre-migration snapshot reads back migrated, like any older record. A write is checked against the read it already makes, before anything lands: a record outside the family is refused with `StackNotFoundError`, the answer `get()` gives, and so is a `restoreVersion()` to a snapshot taken before the record migrated into the family, which is not a version of a record in the collection. A content write to a record stored at a newer version is refused with `MisfitError` before anything is written, since its patch would be validated against a schema the handle does not know. Any other write that leaves a misfit lands, then throws `MisfitError` carrying the stored record, so the app knows the write landed. A deleted record, or a stale `ifVersion`, is refused first, as the untyped write refuses it. diff --git a/packages/core/src/collection.ts b/packages/core/src/collection.ts index f5da81ea..5debbb96 100644 --- a/packages/core/src/collection.ts +++ b/packages/core/src/collection.ts @@ -96,6 +96,13 @@ export type CollectionPage = { records: CollectionRecord[]; misfits: Misfit[]; cursor: string | null; + /** + * A content filter or sort reaches only records at the handle's version. + * True when records at an older version match the rest of the query, so + * the page may be missing records the filter was meant to find. See + * docs/spec/data-model.md § Type handles. + */ + skippedOlder: boolean; }; /** @@ -146,8 +153,8 @@ export type SortableFieldOf = string extends keyof C /** * A collection reads live records of its own family only, as stored, so * none of `typeId`, `baseId`, `includeDeleted` or `presentAt` is offered. - * Content filter keys and the sort field are checked against `C`, the - * handle's version, while matching runs on content as stored. See + * Content filter keys and the sort field are checked against `C`, and + * reach only records stored at the handle's version. See * docs/spec/data-model.md § Type handles. */ export type CollectionQuery = Omit & { @@ -269,6 +276,15 @@ const withoutTypeId = (filter: F | undefined): Omit + (filter?.content !== undefined && Object.keys(filter.content).length > 0) || + (filter?.contentPresent?.length ?? 0) > 0 || + sort?.contentField !== undefined; + /** * The collection over `handle`, shared by `Stack.collection()` and * `ScopedStack.collection()`. `registry` is the opening `Stack`'s, which @@ -360,6 +376,28 @@ export const createCollection = ( return narrow(written, true); }; + /** + * Whether a record stored at an older version matches `filter` without + * its content keys. A scoped page can be empty with a cursor set, so this + * reads on until it finds one or the results end. + */ + const hasOlder = async (filter: RecordFilter): Promise => { + if (handleVersion === 1) return false; + const { content: _content, contentPresent: _present, ...rest } = filter; + const older = Array.from({ length: handleVersion - 1 }, (_, i) => `${handle.baseId}@${i + 1}`); + let cursor: string | undefined; + do { + const page = await client.query({ + filter: { ...rest, typeId: older }, + limit: 1, + ...(cursor && { cursor }), + }); + if (page.records.length > 0) return true; + cursor = page.cursor ?? undefined; + } while (cursor); + return false; + }; + const mutate = async ( id: RecordId, changes: RecordChangeSet, @@ -393,11 +431,18 @@ export const createCollection = ( // Stripped as well as untyped: either would change which records a // page holds, or the shape they arrive in. const { presentAt: _presentAt, filter, ...rest } = query as StackQuery; + const narrowed = readsContent(filter, rest.sort); + const familyFilter = { ...withoutTypeId(filter), baseId: handle.baseId }; const { records, cursor } = await client.query({ ...rest, - filter: { ...withoutTypeId(filter), baseId: handle.baseId }, + filter: narrowed ? { ...familyFilter, typeId: handle.id } : familyFilter, }); - const page: CollectionPage = { records: [], misfits: [], cursor }; + const page: CollectionPage = { + records: [], + misfits: [], + cursor, + skippedOlder: narrowed && (await hasOlder(familyFilter)), + }; for (const stored of records) { const result = classify(stored); if ('misfit' in result) page.misfits.push(result.misfit); diff --git a/packages/core/tests/collection.test.ts b/packages/core/tests/collection.test.ts index 7e3d188e..edec60e0 100644 --- a/packages/core/tests/collection.test.ts +++ b/packages/core/tests/collection.test.ts @@ -269,6 +269,42 @@ describe.each([ expect(records.map((r) => r.content.title)).toEqual(['Emma']); }); + test("a content filter reaches only the handle's version and flags older records", async () => { + const book = await books.create({ title: 'Dune', status: 'finished' }); + await createV1('Emma', 'done'); + await stack.create(BookV3.id, { title: 'Ulysses', status: 'finished' }); + const page = await books.query({ filter: { content: { status: 'finished' } } }); + expect(page.records.map((r) => r.id)).toEqual([book.id]); + expect(page.misfits).toEqual([]); + expect(page.skippedOlder).toBe(true); + }); + + test('a content sort and a presence filter narrow the same way', async () => { + const book = await books.create({ title: 'Dune', status: 'want' }); + await createV1(); + const sorted = await books.query({ sort: { contentField: 'title' } }); + expect(sorted.records.map((r) => r.id)).toEqual([book.id]); + expect(sorted.skippedOlder).toBe(true); + const present = await books.query({ filter: { contentPresent: ['title'] } }); + expect(present.records.map((r) => r.id)).toEqual([book.id]); + expect(present.skippedOlder).toBe(true); + }); + + test('skippedOlder applies the rest of the filter', async () => { + await books.create({ title: 'Dune', status: 'want' }); + await createV1(); + const page = await books.query({ filter: { content: { status: 'want' }, tags: ['fantasy'] } }); + expect(page.skippedOlder).toBe(false); + }); + + test('a query without content keys reads the whole family and flags nothing', async () => { + await books.create({ title: 'Dune', status: 'want' }); + await createV1(); + const page = await books.query({ filter: { content: {}, contentPresent: [] } }); + expect(page.records).toHaveLength(2); + expect(page.skippedOlder).toBe(false); + }); + test('query() drops typeId and presentAt from an untyped caller', async () => { const book = await books.create({ title: 'Dune', status: 'want' }); const newer = await stack.create(BookV3.id, { title: 'Emma', status: 'want' }); From 3c83ed498542e1a8ca19a0b810bce3e7bbaa5f74 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 20:13:53 +0000 Subject: [PATCH 3/4] fix(core): flag newer as well as older records a content query skips Review findings on #447: - A narrowed page now reports records at a newer version too, as skipped.newer, where they were dropped with no signal. The family's versions come from listTypes(), passed in by Stack and ScopedStack, rather than counted down from the handle's version. - The probe is one limit-1 read per side, run beside the page's own; an empty scoped page with a cursor counts as a match instead of paging on. - Path typing covered seven segments where the docs say six. - contentPresent and contentField use ContentPathOf and SortableFieldOf; both print expanded unions in errors, so the inline copy was not needed. - The spec notes that filtering for an unlisted enum value takes a cast. An undefined content value still narrows: every adapter reads it as null. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KQtAGzBB8fDpfqZhc84ZdN --- .changeset/typed-collection-filters.md | 2 +- docs/spec/data-model.md | 4 +- packages/core/src/collection.ts | 90 +++++++++---------- .../core/src/scoped-stack/scoped-stack.ts | 2 +- packages/core/src/stack/stack.ts | 2 +- packages/core/tests/collection.test.ts | 29 ++++-- 6 files changed, 70 insertions(+), 59 deletions(-) diff --git a/.changeset/typed-collection-filters.md b/.changeset/typed-collection-filters.md index f0db7c6f..fcb43aa9 100644 --- a/.changeset/typed-collection-filters.md +++ b/.changeset/typed-collection-filters.md @@ -4,4 +4,4 @@ A collection's `filter.content`, `filter.contentPresent` and `sort.contentField` are typed from its content type: a typo'd field, a value outside an enum or a nested sort field no longer compiles. Adds `ContentPathOf` and `SortableFieldOf`. -A collection query with a content filter or sort reaches only records stored at the handle's version, since a predicate in the handle's shape cannot be translated into an older version's. Its page sets `skippedOlder` when records at an older version match the rest of the query. +A collection query with a content filter or sort reaches only records stored at the handle's version, since a predicate in the handle's shape cannot be translated into another version's. Its page sets `skipped.older` or `skipped.newer` when records at another version match the rest of the query. diff --git a/docs/spec/data-model.md b/docs/spec/data-model.md index 0336dc1a..d8ec3c7f 100644 --- a/docs/spec/data-model.md +++ b/docs/spec/data-model.md @@ -418,7 +418,7 @@ A read types an `enum` field as its listed values or any other string (`'want' | **A misfit is a record of the family the handle cannot represent**: one stored at a version newer than the handle's (`reason: 'newer-version'`), which cannot be migrated down. A misfit is reported, never thrown for a page: -- `query()` returns `{ records, misfits, cursor, skippedOlder }`. Misfits do not count as records, so a page can hold fewer than `limit` records with a cursor still set: `cursor === null` ends the results, a short page does not. The page and its cursor survive one record the app cannot read. +- `query()` returns `{ records, misfits, cursor, skipped }`. Misfits do not count as records, so a page can hold fewer than `limit` records with a cursor still set: `cursor === null` ends the results, a short page does not. The page and its cursor survive one record the app cannot read. - `get()` is a read the app asked for by ID, so it throws `MisfitError`, carrying the `reason` and the record as stored. - `subscribe()` delivers a change whose record does not fit with `misfit` in place of `record`. @@ -426,7 +426,7 @@ A record at an older version that no passed migration carries to the handle's is **A subscription covers the family.** A change's `record`, when present, is migrated to the handle's version; the change's own `typeId` stays the stored one, as `RecordChange.typeId` documents. A change whose record has `deletedAt` set carries neither `record` nor `misfit`, on either surface, and neither does a change that [migrates the record out](#type-migrations) of the family, which the filter matches by its previous `typeId`. The change filter takes every key but `typeId` and `baseId`. -**A content filter or sort reaches only records stored at the handle's version.** The adapter matches and orders content as stored, before any record is migrated, and a migration is an opaque function, so a predicate written in the handle's shape cannot be translated into an older version's. Run across the family, a field a migration renamed, derived or remapped would silently miss the older records or misplace them. So a `query()` whose `filter.content` or `filter.contentPresent` is non-empty, or whose sort is a `contentField`, reads only records whose `typeId` is the handle's, and its page sets `skippedOlder` when records at an older version match the rest of the query: the page may then be missing records the filter was meant to find. Mixed versions are a lasting state, not a backlog, since an app at an older version may keep writing its own, so the app decides what `skippedOlder` means to it; `migrateAll()` clears it only until the next such write. Such a page carries no misfits, since whether a newer record matches is equally unknowable. A query with no content keys reads the whole family. The keys are typed against the handle's version: `filter.content` and `filter.contentPresent` take the content's paths (`ContentPathOf`), through declared objects and arrays, with any suffix below an `open` object or array, and a value of the field's type (an array's element type, since matching is element-wise) or `null`; `sort.contentField` takes only a top-level scalar (`SortableFieldOf`). Path typing stops six segments deep and accepts any suffix below that, a limit of the compiler, not of the [32-segment path](#nested-content-paths). `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. +**A content filter or sort reaches only records stored at the handle's version.** The adapter matches and orders content as stored, before any record is migrated, and a migration is an opaque function, so a predicate written in the handle's shape cannot be translated into an older version's. Run across the family, a field a migration renamed, derived or remapped would silently miss the older records or misplace them. So a `query()` whose `filter.content` or `filter.contentPresent` is non-empty, or whose sort is a `contentField`, reads only records whose `typeId` is the handle's. Its page sets `skipped.older` or `skipped.newer` when records at an older or newer version match the rest of the query: the page may then be missing records the filter was meant to find. Each is one `limit: 1` read, run beside the page's own, and a scoped read that comes back empty with a cursor still set counts as a match rather than reading on, so a flag can be set when nothing was skipped but is never clear when something was. Mixed versions are a lasting state, not a backlog, since an app at an older version may keep writing its own, so the app decides what `skipped.older` means to it; `migrateAll()` clears it only until the next such write. `skipped.newer` says the app is behind, which an unfiltered page reports as misfits; a narrowed page carries none. A query with no content keys reads the whole family. The keys are typed against the handle's version: `filter.content` and `filter.contentPresent` take the content's paths (`ContentPathOf`), through declared objects and arrays, with any suffix below an `open` object or array, and a value of the field's type as a write takes it (an array's element type, since matching is element-wise) or `null`, so an enum value the handle does not list is filtered for with a cast, the price of refusing a misspelled one; `sort.contentField` takes only a top-level scalar (`SortableFieldOf`). Path typing stops six segments deep and accepts any suffix below that, a limit of the compiler, not of the [32-segment path](#nested-content-paths). `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. **Every verb that returns a record returns it as `get()` would.** The write already returned the record, so narrowing it costs no round trip, and a `restoreVersion()` to a pre-migration snapshot reads back migrated, like any older record. A write is checked against the read it already makes, before anything lands: a record outside the family is refused with `StackNotFoundError`, the answer `get()` gives, and so is a `restoreVersion()` to a snapshot taken before the record migrated into the family, which is not a version of a record in the collection. A content write to a record stored at a newer version is refused with `MisfitError` before anything is written, since its patch would be validated against a schema the handle does not know. Any other write that leaves a misfit lands, then throws `MisfitError` carrying the stored record, so the app knows the write landed. A deleted record, or a stale `ifVersion`, is refused first, as the untyped write refuses it. diff --git a/packages/core/src/collection.ts b/packages/core/src/collection.ts index 5debbb96..5d69ef1a 100644 --- a/packages/core/src/collection.ts +++ b/packages/core/src/collection.ts @@ -39,6 +39,8 @@ import type { RecordFilter, RecordId, StackQuery, + StackType, + TypeId, StackRecord, SubscribeOptions, Unsubscribe, @@ -98,11 +100,11 @@ export type CollectionPage = { cursor: string | null; /** * A content filter or sort reaches only records at the handle's version. - * True when records at an older version match the rest of the query, so - * the page may be missing records the filter was meant to find. See - * docs/spec/data-model.md § Type handles. + * Each flag is set when records at an older or newer version match, or + * may match, the rest of the query: the page may be missing records the + * filter was meant to find. See docs/spec/data-model.md § Type handles. */ - skippedOlder: boolean; + skipped: { older: boolean; newer: boolean }; }; /** @@ -111,7 +113,7 @@ export type CollectionPage = { * A typing limit, not a query limit: see docs/spec/data-model.md § Nested * content paths. */ -type PathDepth = [never, 0, 1, 2, 3, 4, 5]; +type PathDepth = [never, 0, 1, 2, 3, 4]; /** * Each content path of `V` reached at `P`, paired with the value a filter @@ -141,7 +143,7 @@ type FieldEntry = [D] extends [never * objects and arrays. Below an `open` object or array, or six segments * deep, any suffix is a path. */ -export type ContentPathOf = FieldEntry[0]; +export type ContentPathOf = FieldEntry[0]; /** The top-level scalar fields of `C`, the ones a content sort reaches. */ export type SortableFieldOf = string extends keyof C @@ -162,21 +164,14 @@ export type CollectionQuery = Omit & { - content?: { [E in FieldEntry as E[0]]?: E[1] | null }; - contentPresent?: FieldEntry[0][]; + content?: { [E in FieldEntry as E[0]]?: E[1] | null }; + contentPresent?: ContentPathOf[]; }; sort?: | { field: NativeSortField; contentField?: never; direction?: 'asc' | 'desc' } | { field?: never; - // Spelled out rather than SortableFieldOf, so an error lists the fields. - contentField: string extends keyof C - ? string - : { - [K in keyof C & string]-?: Required[K] extends string | number | boolean - ? K - : never; - }[keyof C & string]; + contentField: SortableFieldOf; direction?: 'asc' | 'desc'; }; }; @@ -281,19 +276,21 @@ const withoutTypeId = (filter: F | undefined): Omit - (filter?.content !== undefined && Object.keys(filter.content).length > 0) || + Object.keys(filter?.content ?? {}).length > 0 || (filter?.contentPresent?.length ?? 0) > 0 || sort?.contentField !== undefined; /** * The collection over `handle`, shared by `Stack.collection()` and * `ScopedStack.collection()`. `registry` is the opening `Stack`'s, which - * the app's own process filled. + * the app's own process filled. `listTypes` names the family's versions, + * which `StackClient` does not offer. */ export const createCollection = ( client: StackClient, registry: MigrationRegistry, handle: TypeHandle, + listTypes: () => Promise, ): Collection> => { type C = ContentOf; if (handle.migratesFrom !== undefined && !registry.path(handle.migratesFrom, handle.id)) { @@ -377,25 +374,30 @@ export const createCollection = ( }; /** - * Whether a record stored at an older version matches `filter` without - * its content keys. A scoped page can be empty with a cursor set, so this - * reads on until it finds one or the results end. + * Whether a record stored at one of `typeIds` matches, or may match, + * `filter` without its content keys. A scoped page can be empty with a + * cursor set, and reading on is unbounded, so that page counts as a match. */ - const hasOlder = async (filter: RecordFilter): Promise => { - if (handleVersion === 1) return false; + const anyAt = async (typeIds: TypeId[], filter: RecordFilter): Promise => { + if (typeIds.length === 0) return false; const { content: _content, contentPresent: _present, ...rest } = filter; - const older = Array.from({ length: handleVersion - 1 }, (_, i) => `${handle.baseId}@${i + 1}`); - let cursor: string | undefined; - do { - const page = await client.query({ - filter: { ...rest, typeId: older }, - limit: 1, - ...(cursor && { cursor }), - }); - if (page.records.length > 0) return true; - cursor = page.cursor ?? undefined; - } while (cursor); - return false; + const page = await client.query({ filter: { ...rest, typeId: typeIds }, limit: 1 }); + return page.records.length > 0 || page.cursor !== null; + }; + + const skippedVersions = async (filter: RecordFilter): Promise['skipped']> => { + const family = (await listTypes()).filter((t) => t.baseId === handle.baseId); + const [older, newer] = await Promise.all([ + anyAt( + family.filter((t) => versionOf(t.id) < handleVersion).map((t) => t.id), + filter, + ), + anyAt( + family.filter((t) => versionOf(t.id) > handleVersion).map((t) => t.id), + filter, + ), + ]); + return { older, newer }; }; const mutate = async ( @@ -433,16 +435,14 @@ export const createCollection = ( const { presentAt: _presentAt, filter, ...rest } = query as StackQuery; const narrowed = readsContent(filter, rest.sort); const familyFilter = { ...withoutTypeId(filter), baseId: handle.baseId }; - const { records, cursor } = await client.query({ - ...rest, - filter: narrowed ? { ...familyFilter, typeId: handle.id } : familyFilter, - }); - const page: CollectionPage = { - records: [], - misfits: [], - cursor, - skippedOlder: narrowed && (await hasOlder(familyFilter)), - }; + const [{ records, cursor }, skipped] = await Promise.all([ + client.query({ + ...rest, + filter: narrowed ? { ...familyFilter, typeId: handle.id } : familyFilter, + }), + narrowed ? skippedVersions(familyFilter) : { older: false, newer: false }, + ]); + const page: CollectionPage = { records: [], misfits: [], cursor, skipped }; for (const stored of records) { const result = classify(stored); if ('misfit' in result) page.misfits.push(result.misfit); diff --git a/packages/core/src/scoped-stack/scoped-stack.ts b/packages/core/src/scoped-stack/scoped-stack.ts index 93e105c6..1fd28a4c 100644 --- a/packages/core/src/scoped-stack/scoped-stack.ts +++ b/packages/core/src/scoped-stack/scoped-stack.ts @@ -186,7 +186,7 @@ export class ScopedStack implements StackClient { collection(handle: TypeHandle): Collection> { this.assertStackOpen(); - return createCollection(this, this.migrations, handle); + return createCollection(this, this.migrations, handle, () => this.stack.listTypes()); } /** diff --git a/packages/core/src/stack/stack.ts b/packages/core/src/stack/stack.ts index 618b3bef..e2cf121a 100644 --- a/packages/core/src/stack/stack.ts +++ b/packages/core/src/stack/stack.ts @@ -395,7 +395,7 @@ export class Stack implements StackClient { collection(handle: TypeHandle): Collection> { this.assertOpen(); - return createCollection(this, this.migrations, handle); + return createCollection(this, this.migrations, handle, () => this.listTypes()); } // ------------------------------------------------------- diff --git a/packages/core/tests/collection.test.ts b/packages/core/tests/collection.test.ts index edec60e0..4e647afe 100644 --- a/packages/core/tests/collection.test.ts +++ b/packages/core/tests/collection.test.ts @@ -269,14 +269,14 @@ describe.each([ expect(records.map((r) => r.content.title)).toEqual(['Emma']); }); - test("a content filter reaches only the handle's version and flags older records", async () => { + test("a content filter reaches only the handle's version and flags both sides", async () => { const book = await books.create({ title: 'Dune', status: 'finished' }); await createV1('Emma', 'done'); await stack.create(BookV3.id, { title: 'Ulysses', status: 'finished' }); const page = await books.query({ filter: { content: { status: 'finished' } } }); expect(page.records.map((r) => r.id)).toEqual([book.id]); expect(page.misfits).toEqual([]); - expect(page.skippedOlder).toBe(true); + expect(page.skipped).toEqual({ older: true, newer: true }); }); test('a content sort and a presence filter narrow the same way', async () => { @@ -284,17 +284,18 @@ describe.each([ await createV1(); const sorted = await books.query({ sort: { contentField: 'title' } }); expect(sorted.records.map((r) => r.id)).toEqual([book.id]); - expect(sorted.skippedOlder).toBe(true); + expect(sorted.skipped).toEqual({ older: true, newer: false }); const present = await books.query({ filter: { contentPresent: ['title'] } }); expect(present.records.map((r) => r.id)).toEqual([book.id]); - expect(present.skippedOlder).toBe(true); + expect(present.skipped).toEqual({ older: true, newer: false }); }); - test('skippedOlder applies the rest of the filter', async () => { + test('skipped applies the rest of the filter', async () => { await books.create({ title: 'Dune', status: 'want' }); await createV1(); + await stack.create(BookV3.id, { title: 'Ulysses', status: 'want' }); const page = await books.query({ filter: { content: { status: 'want' }, tags: ['fantasy'] } }); - expect(page.skippedOlder).toBe(false); + expect(page.skipped).toEqual({ older: false, newer: false }); }); test('a query without content keys reads the whole family and flags nothing', async () => { @@ -302,7 +303,15 @@ describe.each([ await createV1(); const page = await books.query({ filter: { content: {}, contentPresent: [] } }); expect(page.records).toHaveLength(2); - expect(page.skippedOlder).toBe(false); + expect(page.skipped).toEqual({ older: false, newer: false }); + }); + + test('an undefined content value is a filter, as an adapter reads it', async () => { + await books.create({ title: 'Dune', status: 'want' }); + await createV1(); + const page = await books.query({ filter: { content: { pages: undefined } } }); + expect(page.records).toHaveLength(1); + expect(page.skipped).toEqual({ older: true, newer: false }); }); test('query() drops typeId and presentAt from an untyped caller', async () => { @@ -737,10 +746,12 @@ describe('content filter and sort typing', () => { }>; }> >; - const ok: DeepQ = { filter: { content: { 'a.b.c.d.e': {}, 'a.b.c.d.e.f.g.anything': 1 } } }; + const ok: DeepQ = { filter: { content: { 'a.b.c.d.e': {}, 'a.b.c.d.e.f.anything': 1 } } }; + // @ts-expect-error the sixth segment is still typed + const sixth: DeepQ = { filter: { content: { 'a.b.c.d.e.x': 1 } } }; // @ts-expect-error a typo above the limit is still caught const typo: DeepQ = { filter: { content: { 'a.x': 1 } } }; - expect([ok, typo]).toHaveLength(2); + expect([ok, sixth, typo]).toHaveLength(3); }); test('an untyped handle takes any path and sort field', () => { From 5ca55b056c1f21547a9d6de7d809e099f8fd37b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 14:28:42 +0000 Subject: [PATCH 4/4] fix(core): refuse object filter values and probe skipped versions once Second review on #447: - A filter value at a path reaching an object compiled but never matched, since adapters compare the value a path reaches. Such a path now takes only null, and a value below an open node any scalar. - skipped is computed on the first page only; it depends on the query. - The narrowed read and the probes pass concrete typeIds without baseId, so the family's types are listed once per first page rather than on every read underneath. - Family versions come from StackType.version in one pass. - The spec says the untyped query() still matches across the family. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KQtAGzBB8fDpfqZhc84ZdN --- .changeset/typed-collection-filters.md | 4 +- docs/spec/data-model.md | 4 +- packages/core/src/collection.ts | 67 +++++++++++++++++--------- packages/core/tests/collection.test.ts | 34 ++++++++++++- 4 files changed, 79 insertions(+), 30 deletions(-) diff --git a/.changeset/typed-collection-filters.md b/.changeset/typed-collection-filters.md index fcb43aa9..a8fcc330 100644 --- a/.changeset/typed-collection-filters.md +++ b/.changeset/typed-collection-filters.md @@ -2,6 +2,6 @@ '@haverstack/core': minor --- -A collection's `filter.content`, `filter.contentPresent` and `sort.contentField` are typed from its content type: a typo'd field, a value outside an enum or a nested sort field no longer compiles. Adds `ContentPathOf` and `SortableFieldOf`. +A collection's `filter.content`, `filter.contentPresent` and `sort.contentField` are typed from its content type: a typo'd field, a value outside an enum, an object value (which never matches) or a nested sort field no longer compiles. Adds `ContentPathOf` and `SortableFieldOf`. -A collection query with a content filter or sort reaches only records stored at the handle's version, since a predicate in the handle's shape cannot be translated into another version's. Its page sets `skipped.older` or `skipped.newer` when records at another version match the rest of the query. +A collection query with a content filter or sort reaches only records stored at the handle's version, since a predicate in the handle's shape cannot be translated into another version's. Its first page sets `skipped.older` or `skipped.newer` when records at another version match the rest of the query. diff --git a/docs/spec/data-model.md b/docs/spec/data-model.md index d8ec3c7f..af8e6df9 100644 --- a/docs/spec/data-model.md +++ b/docs/spec/data-model.md @@ -418,7 +418,7 @@ A read types an `enum` field as its listed values or any other string (`'want' | **A misfit is a record of the family the handle cannot represent**: one stored at a version newer than the handle's (`reason: 'newer-version'`), which cannot be migrated down. A misfit is reported, never thrown for a page: -- `query()` returns `{ records, misfits, cursor, skipped }`. Misfits do not count as records, so a page can hold fewer than `limit` records with a cursor still set: `cursor === null` ends the results, a short page does not. The page and its cursor survive one record the app cannot read. +- `query()` returns `{ records, misfits, cursor }`, and `skipped` on the first page. Misfits do not count as records, so a page can hold fewer than `limit` records with a cursor still set: `cursor === null` ends the results, a short page does not. The page and its cursor survive one record the app cannot read. - `get()` is a read the app asked for by ID, so it throws `MisfitError`, carrying the `reason` and the record as stored. - `subscribe()` delivers a change whose record does not fit with `misfit` in place of `record`. @@ -426,7 +426,7 @@ A record at an older version that no passed migration carries to the handle's is **A subscription covers the family.** A change's `record`, when present, is migrated to the handle's version; the change's own `typeId` stays the stored one, as `RecordChange.typeId` documents. A change whose record has `deletedAt` set carries neither `record` nor `misfit`, on either surface, and neither does a change that [migrates the record out](#type-migrations) of the family, which the filter matches by its previous `typeId`. The change filter takes every key but `typeId` and `baseId`. -**A content filter or sort reaches only records stored at the handle's version.** The adapter matches and orders content as stored, before any record is migrated, and a migration is an opaque function, so a predicate written in the handle's shape cannot be translated into an older version's. Run across the family, a field a migration renamed, derived or remapped would silently miss the older records or misplace them. So a `query()` whose `filter.content` or `filter.contentPresent` is non-empty, or whose sort is a `contentField`, reads only records whose `typeId` is the handle's. Its page sets `skipped.older` or `skipped.newer` when records at an older or newer version match the rest of the query: the page may then be missing records the filter was meant to find. Each is one `limit: 1` read, run beside the page's own, and a scoped read that comes back empty with a cursor still set counts as a match rather than reading on, so a flag can be set when nothing was skipped but is never clear when something was. Mixed versions are a lasting state, not a backlog, since an app at an older version may keep writing its own, so the app decides what `skipped.older` means to it; `migrateAll()` clears it only until the next such write. `skipped.newer` says the app is behind, which an unfiltered page reports as misfits; a narrowed page carries none. A query with no content keys reads the whole family. The keys are typed against the handle's version: `filter.content` and `filter.contentPresent` take the content's paths (`ContentPathOf`), through declared objects and arrays, with any suffix below an `open` object or array, and a value of the field's type as a write takes it (an array's element type, since matching is element-wise) or `null`, so an enum value the handle does not list is filtered for with a cast, the price of refusing a misspelled one; `sort.contentField` takes only a top-level scalar (`SortableFieldOf`). Path typing stops six segments deep and accepts any suffix below that, a limit of the compiler, not of the [32-segment path](#nested-content-paths). `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. +**A content filter or sort reaches only records stored at the handle's version.** The adapter matches and orders content as stored, before any record is migrated, and a migration is an opaque function, so a predicate written in the handle's shape cannot be translated into an older version's. Run across the family, a field a migration renamed, derived or remapped would silently miss the older records or misplace them. So a `query()` whose `filter.content` or `filter.contentPresent` is non-empty, or whose sort is a `contentField`, reads only records whose `typeId` is the handle's, even for a field no migration touched, since core cannot tell which fields those are. The untyped `query({ filter: { baseId, content } })` still matches across the family, returning records as stored. The first page of such a query sets `skipped.older` or `skipped.newer` when records at an older or newer version match the rest of the query: the results may then be missing records the filter was meant to find. A later page carries no `skipped`, since the flags depend on the query, not the page. Each is one `limit: 1` read, run beside the first page's own, and a scoped read that comes back empty with a cursor still set counts as a match rather than reading on, so a flag can be set when nothing was skipped but is never clear when something was. Mixed versions are a lasting state, not a backlog, since an app at an older version may keep writing its own, so the app decides what `skipped.older` means to it; `migrateAll()` clears it only until the next such write. `skipped.newer` says the app is behind, which an unfiltered page reports as misfits; a narrowed page carries none. A query with no content keys reads the whole family. The keys are typed against the handle's version: `filter.content` and `filter.contentPresent` take the content's paths (`ContentPathOf`), through declared objects and arrays, with any suffix below an `open` object or array, and a value of the field's type as a write takes it (an array's element type, since matching is element-wise) or `null`. A path reaching an object takes only `null`, since an adapter compares the value a path reaches and an object never equals one; a value below an `open` node is any scalar. Typing values as writes take them means an enum value the handle does not list is filtered for with a cast, the price of refusing a misspelled one; `sort.contentField` takes only a top-level scalar (`SortableFieldOf`). Path typing stops six segments deep and accepts any suffix below that, a limit of the compiler, not of the [32-segment path](#nested-content-paths). `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. **Every verb that returns a record returns it as `get()` would.** The write already returned the record, so narrowing it costs no round trip, and a `restoreVersion()` to a pre-migration snapshot reads back migrated, like any older record. A write is checked against the read it already makes, before anything lands: a record outside the family is refused with `StackNotFoundError`, the answer `get()` gives, and so is a `restoreVersion()` to a snapshot taken before the record migrated into the family, which is not a version of a record in the collection. A content write to a record stored at a newer version is refused with `MisfitError` before anything is written, since its patch would be validated against a schema the handle does not know. Any other write that leaves a misfit lands, then throws `MisfitError` carrying the stored record, so the app knows the write landed. A deleted record, or a stale `ifVersion`, is refused first, as the untyped write refuses it. diff --git a/packages/core/src/collection.ts b/packages/core/src/collection.ts index 5d69ef1a..44118ba7 100644 --- a/packages/core/src/collection.ts +++ b/packages/core/src/collection.ts @@ -101,10 +101,12 @@ export type CollectionPage = { /** * A content filter or sort reaches only records at the handle's version. * Each flag is set when records at an older or newer version match, or - * may match, the rest of the query: the page may be missing records the - * filter was meant to find. See docs/spec/data-model.md § Type handles. + * may match, the rest of the query: the results may be missing records + * the filter was meant to find. On the first page only, since it depends + * on the query rather than the page. See docs/spec/data-model.md § Type + * handles. */ - skipped: { older: boolean; newer: boolean }; + skipped?: { older: boolean; newer: boolean }; }; /** @@ -115,25 +117,29 @@ export type CollectionPage = { */ type PathDepth = [never, 0, 1, 2, 3, 4]; +/** What a filter value can equal: an adapter compares the value a path reaches. */ +type Leaf = string | number | boolean; + /** * Each content path of `V` reached at `P`, paired with the value a filter * compares there. An array is matched element-wise, so it is its element. + * An object never equals a filter value, so its path takes only `null`. */ type PathEntry

= [D] extends [never] - ? [P | `${P}.${string}`, unknown] + ? [P | `${P}.${string}`, Leaf] : V extends readonly (infer E)[] ? unknown extends E - ? [P | `${P}.${string}`, unknown] + ? [P | `${P}.${string}`, Leaf] : PathEntry : V extends object ? string extends keyof V - ? [P, V] | [`${P}.${string}`, unknown] - : [P, V] | FieldEntry + ? [P, null] | [`${P}.${string}`, Leaf] + : [P, null] | FieldEntry : [P, V]; /** `Required[K]` rather than `NonNullable`, which would leak into errors. */ type FieldEntry = [D] extends [never] - ? [`${Prefix}${string}`, unknown] + ? [`${Prefix}${string}`, Leaf] : { [K in keyof C & string]-?: PathEntry<`${Prefix}${K}`, Required[K], D>; }[keyof C & string]; @@ -385,19 +391,20 @@ export const createCollection = ( return page.records.length > 0 || page.cursor !== null; }; - const skippedVersions = async (filter: RecordFilter): Promise['skipped']> => { - const family = (await listTypes()).filter((t) => t.baseId === handle.baseId); - const [older, newer] = await Promise.all([ - anyAt( - family.filter((t) => versionOf(t.id) < handleVersion).map((t) => t.id), - filter, - ), - anyAt( - family.filter((t) => versionOf(t.id) > handleVersion).map((t) => t.id), - filter, - ), + const skippedVersions = async ( + filter: RecordFilter, + ): Promise['skipped']>> => { + const older: TypeId[] = []; + const newer: TypeId[] = []; + for (const type of await listTypes()) { + if (type.baseId !== handle.baseId || type.version === handleVersion) continue; + (type.version < handleVersion ? older : newer).push(type.id); + } + const [skippedOlder, skippedNewer] = await Promise.all([ + anyAt(older, filter), + anyAt(newer, filter), ]); - return { older, newer }; + return { older: skippedOlder, newer: skippedNewer }; }; const mutate = async ( @@ -434,15 +441,27 @@ export const createCollection = ( // page holds, or the shape they arrive in. const { presentAt: _presentAt, filter, ...rest } = query as StackQuery; const narrowed = readsContent(filter, rest.sort); - const familyFilter = { ...withoutTypeId(filter), baseId: handle.baseId }; + // A concrete typeId needs no family lookup; baseId does. + const { baseId: _baseId, ...unversioned } = withoutTypeId(filter); const [{ records, cursor }, skipped] = await Promise.all([ client.query({ ...rest, - filter: narrowed ? { ...familyFilter, typeId: handle.id } : familyFilter, + filter: narrowed + ? { ...unversioned, typeId: handle.id } + : { ...unversioned, baseId: handle.baseId }, }), - narrowed ? skippedVersions(familyFilter) : { older: false, newer: false }, + rest.cursor !== undefined + ? undefined + : narrowed + ? skippedVersions(unversioned) + : { older: false, newer: false }, ]); - const page: CollectionPage = { records: [], misfits: [], cursor, skipped }; + const page: CollectionPage = { + records: [], + misfits: [], + cursor, + ...(skipped && { skipped }), + }; for (const stored of records) { const result = classify(stored); if ('misfit' in result) page.misfits.push(result.misfit); diff --git a/packages/core/tests/collection.test.ts b/packages/core/tests/collection.test.ts index 4e647afe..e9c45a26 100644 --- a/packages/core/tests/collection.test.ts +++ b/packages/core/tests/collection.test.ts @@ -290,6 +290,21 @@ describe.each([ expect(present.skipped).toEqual({ older: true, newer: false }); }); + test('skipped is reported on the first page only', async () => { + await books.create({ title: 'Dune', status: 'want' }); + await books.create({ title: 'Emma', status: 'want' }); + await createV1(); + const first = await books.query({ filter: { content: { status: 'want' } }, limit: 1 }); + expect(first.skipped).toEqual({ older: true, newer: false }); + const second = await books.query({ + filter: { content: { status: 'want' } }, + limit: 1, + cursor: first.cursor!, + }); + expect(second.records).toHaveLength(1); + expect(second).not.toHaveProperty('skipped'); + }); + test('skipped applies the rest of the filter', async () => { await books.create({ title: 'Dune', status: 'want' }); await createV1(); @@ -720,9 +735,24 @@ describe('content filter and sort typing', () => { const badNested: Q = { filter: { content: { 'address.geo.lat': 'north' } } }; // @ts-expect-error a path through a scalar const throughScalar: Q = { filter: { content: { 'name.first': 'Ada' } } }; + // @ts-expect-error an object never equals a filter value + const object: Q = { filter: { content: { address: { city: 'Lisbon' } } } }; + // @ts-expect-error nor does an element of an array of objects + const element: Q = { filter: { content: { emails: { value: 'ada@example.com' } } } }; + // @ts-expect-error nor an open object itself + const open: Q = { filter: { content: { extra: {} } } }; // @ts-expect-error a typo'd presence path const badPresent: Q = { filter: { contentPresent: ['adress.city'] } }; - expect([typo, badEnum, badNested, throughScalar, badPresent]).toHaveLength(5); + expect([ + typo, + badEnum, + badNested, + throughScalar, + object, + element, + open, + badPresent, + ]).toHaveLength(8); }); test('sorts by top-level scalars only', () => { @@ -746,7 +776,7 @@ describe('content filter and sort typing', () => { }>; }> >; - const ok: DeepQ = { filter: { content: { 'a.b.c.d.e': {}, 'a.b.c.d.e.f.anything': 1 } } }; + const ok: DeepQ = { filter: { content: { 'a.b.c.d.e': null, 'a.b.c.d.e.f.anything': 1 } } }; // @ts-expect-error the sixth segment is still typed const sixth: DeepQ = { filter: { content: { 'a.b.c.d.e.x': 1 } } }; // @ts-expect-error a typo above the limit is still caught