From a40cee01aaaf37baf95be802f266ff1ee6ce320d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 18:36:23 +0000 Subject: [PATCH 1/2] feat(core): declarative migrations, so content queries span a family's versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A migration is now a list of declared steps — rename, map, add, drop, derive — instead of a function. migration() checks them against both handles' schemas when it runs, and keeps the field origins they imply. A collection uses those origins to carry a content filter or sort into each older version's shape, sent as filter.contentByType and sort.contentFieldByType. A version that does not translate is narrowed and reported in skipped.older, as before. Both shapes are capped by new capabilities; local adapters declare 64, and a server declaring 0 gets the handle-only query. Spec, conformance fixtures, MemoryAdapter, sqlite-shared and the wire parser move together. Closes #448 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LREkawquacdfohXcxKRYWa --- .changeset/declarative-migrations.md | 12 + README.md | 16 +- docs/spec/adapters.md | 6 +- docs/spec/apps.md | 4 +- docs/spec/data-model.md | 99 ++++- docs/spec/versioning.md | 2 +- docs/spec/wire-format.md | 18 +- packages/adapter-api/tests/api.test.ts | 42 +- packages/adapter-api/tests/helpers.ts | 8 +- packages/adapter-conformance/src/record.ts | 184 +++++++++ .../conformance-fixtures/src/discovery.ts | 18 +- packages/conformance-fixtures/src/errors.ts | 38 ++ packages/conformance-fixtures/src/query.ts | 96 +++++ packages/core/README.md | 16 +- packages/core/src/adapter/index.ts | 8 +- packages/core/src/adapter/sort.ts | 19 + packages/core/src/collection.ts | 216 ++++++++--- packages/core/src/content-match.ts | 61 +++ packages/core/src/content-translation.ts | 132 +++++++ packages/core/src/index.ts | 11 + packages/core/src/migration-steps.ts | 365 ++++++++++++++++++ packages/core/src/query-validation.ts | 182 +++++++-- packages/core/src/stack/migrations.ts | 20 + packages/core/src/stack/stack.ts | 2 + packages/core/src/testing.ts | 92 ++--- packages/core/src/type-handle.ts | 43 ++- packages/core/src/types/capabilities.ts | 13 +- packages/core/src/types/query.ts | 53 ++- packages/core/src/wire/request.ts | 75 +++- packages/core/tests/change-events.test.ts | 2 +- packages/core/tests/collection.test.ts | 294 ++++++++++++-- packages/core/tests/combine.test.ts | 2 + packages/core/tests/install.test.ts | 3 +- packages/core/tests/migration-steps.test.ts | 86 +++++ .../core/tests/query-capabilities.test.ts | 144 ++++++- packages/core/tests/stack.test.ts | 87 ++--- packages/core/tests/type-handle.test.ts | 219 +++++++++-- packages/core/tests/wire-request.test.ts | 35 ++ .../tests/record.test.ts | 2 + packages/sqlite-shared/src/cursor.ts | 40 +- packages/sqlite-shared/src/query.ts | 221 ++++++++--- packages/sqlite-shared/src/record-adapter.ts | 4 + packages/wire-types/src/discovery.ts | 19 +- packages/wire-types/tests/changes.test.ts | 2 + packages/wire-types/tests/discovery.test.ts | 34 +- 45 files changed, 2667 insertions(+), 378 deletions(-) create mode 100644 .changeset/declarative-migrations.md create mode 100644 packages/core/src/content-match.ts create mode 100644 packages/core/src/content-translation.ts create mode 100644 packages/core/src/migration-steps.ts create mode 100644 packages/core/tests/migration-steps.test.ts diff --git a/.changeset/declarative-migrations.md b/.changeset/declarative-migrations.md new file mode 100644 index 00000000..e7a1b459 --- /dev/null +++ b/.changeset/declarative-migrations.md @@ -0,0 +1,12 @@ +--- +'@haverstack/core': minor +'@haverstack/wire-types': minor +'@haverstack/conformance-fixtures': minor +'@haverstack/adapter-conformance': minor +'@haverstack/record-adapter-sqlite': minor +'@haverstack/record-adapter-do-sqlite': minor +--- + +Migrations are declared steps. `migration(from, to, steps)` takes a list built from `rename`, `map`, `add`, `drop` and `derive`, replacing the function form, and checks the list against both handles' schemas when it runs: a step naming a field that is not there, a carried field the target would refuse, or a required field nothing produces throws `StackMigrationError`. `Migration` carries its `steps`. + +A collection's content filter or sort now spans the older versions its migrations translate it to, rather than only the handle's version. A key across a `rename` or `add` translates; one across a `derive`, or a sort across a `map`, narrows that version and sets `skipped.older` as before. Two query shapes carry it, each gated by a new capability that is a cap on keys: `filter.contentByType` (a content predicate per `typeId`, never beside `content` or `contentPresent`) and `sort.contentFieldByType` (a sort field or a reduced constant per `typeId`). `StackCapabilities` gains `filter.contentByType` and `sort.contentFieldByType`; local adapters declare 64, and `normalizeCapabilities()` reads an absent or malformed cap as 0. `parseQueryBody()` accepts both shapes. Conformance fixtures pin both shapes and their 400s, and the adapter conformance suite checks them. diff --git a/README.md b/README.md index 81813265..a8d4b83a 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ An app has two layers, because the stack draws a line between them: Migrations belong to neither. They are passed to `Stack.open()` on **every startup**, so the chain is complete before the first read. ```ts -import { Stack, migration, typeHandle } from '@haverstack/core'; +import { Stack, add, migration, typeHandle } from '@haverstack/core'; import type { Collection, ContentOf, StackClient } from '@haverstack/core'; const NoteV1 = typeHandle({ @@ -212,11 +212,9 @@ export const Note = typeHandle({ }, }); -// Startup: pass to every Stack.open(). `content` is NoteV1's, and the -// result must be Note's. -export const noteMigrations = [ - migration(NoteV1, Note, (content) => ({ ...content, pinned: false })), -]; +// Startup: pass to every Stack.open(). Steps are checked against both +// handles' schemas when migration() runs. +export const noteMigrations = [migration(NoteV1, Note, [add('pinned', false)])]; // Data layer: whoever the stack lets in. export class Notes { @@ -314,7 +312,7 @@ A relationship's `target` says which identifier space its value lives in — a R ### Migrations -Types can evolve over time. A new version names the one it migrates from, and `migration()` builds the step between two handles, typed on both. Pass every step to `Stack.open()`; the library composes adjacent steps into chains: +Types can evolve over time. A new version names the one it migrates from, and `migration()` builds the step between two handles from declared steps — `rename`, `map`, `add`, `drop` and `derive` — checked against both schemas when it runs. Pass every step to `Stack.open()`; the library composes adjacent steps into chains: ```ts const NoteV2 = typeHandle({ @@ -325,12 +323,12 @@ const NoteV2 = typeHandle({ }); const stack = await Stack.open(adapter, { - migrations: [migration(NoteV1, NoteV2, (content) => ({ ...content, title: '' }))], + migrations: [migration(NoteV1, NoteV2, [add('title', '')])], }); await stack.defineType(NoteV2); ``` -Records stay at the version they were written at: `get()` and `query()` return them as stored, `presentAt: 'latest'` migrates them in memory for one read, and `stack.migrateAll()` commits a family to disk. Pass migrations at every startup — see [Writing an app](#writing-an-app). +Records stay at the version they were written at: `get()` and `query()` return them as stored, `presentAt: 'latest'` migrates them in memory for one read, and `stack.migrateAll()` commits a family to disk. Because the steps are declared rather than a function, a collection carries a content filter or sort across the versions they translate to — a filter on `title` reaches records still at `@1`. Pass migrations at every startup — see [Writing an app](#writing-an-app). ### History, and watching for changes diff --git a/docs/spec/adapters.md b/docs/spec/adapters.md index cc9dbfad..bc2a5b79 100644 --- a/docs/spec/adapters.md +++ b/docs/spec/adapters.md @@ -154,11 +154,13 @@ type StackCapabilities = { filter: { content: 'none' | 'field' | 'path'; // how far a content filter key may reach contentPresent: boolean; // filter.contentPresent is honored + contentByType: number; // most arms filter.contentByType may hold; 0 = not honored search: boolean; // filter.search is honored }; sort: { fields: ('createdAt' | 'updatedAt' | 'version')[]; // native columns this adapter can order by contentField: boolean; // sort.contentField is honored + contentFieldByType: number; // most keys sort.contentFieldByType may hold; 0 = not honored }; limits: { attachmentBytes: number | null; // upload size ceiling, or null = client can't pre-check @@ -179,11 +181,13 @@ One type, `StackCapabilities`, names both sides: an adapter declares it as `capa **`filter.contentPresent` sits beside the ladder rather than on it.** It gates `filter.contentPresent`, and is required `true` for local adapters on the same reasoning: asking whether a path holds a value is the same walk with a weaker predicate. It is not a rung because a server promising to match a content value has not thereby promised to answer whether one is there at all. It is meaningful only above `'none'` — presence travels in the `POST /records/query` body, which a server reaching no content does not expose — and a presence path is itself gated by the rung, so a multi-segment one needs `'path'`. +**`filter.contentByType` and `sort.contentFieldByType` are caps, not flags.** Each is the most keys the [per-Type shape](./data-model.md#content-by-type) may hold, and `0` declines it. A local adapter must honor both, on the reasoning that makes `'path'` required: an arm is the content filter it already runs, scoped to one `typeId`. The first-party ones declare `64`, far above any family's version count and inside SQLite's limits on expression depth and compound statements. A server may declare any cap, `0` included, and a collection then narrows the versions it cannot reach rather than failing the query — see [Data model § Content queries across versions](./data-model.md#content-queries-across-versions). Both are meaningful only above `filter.content: 'none'`, since they travel in the `POST /records/query` body. + **`filter.search` has no local-required rule.** A full-text index is a different mechanism from field matching, and neither implies the other, so a local adapter may legitimately decline it (see the JSON adapter note below). **`sort.contentField` is a boolean, and `sort.fields` names native columns only.** Content fields are app-defined and unbounded, so an adapter cannot enumerate them; and an adapter that indexes content for sorting indexes [every top-level scalar](./data-model.md#sorting-by-a-content-field), so there is nothing per-field left to declare. `sort.contentField` is required `true` for local adapters, and independent of `filter.content`: the two describe ordering and filtering, and a server may offer either without the other — filtering is answerable at whatever scale a server already scans at, where ordering wants an index. Both are enforced: `Stack.query()` refuses a sort the adapter hasn't declared rather than letting it be answered in some other order, which a caller reading one bounded page cannot detect. -**Absent, malformed or unrecognized reads as the least capable value it could stand for.** One rule, for every entry: `'none'` for the reach, `false` for a flag, nothing for `sort.fields`, `null` for a limit. A discovery response that omits a group, omits `capabilities` entirely, or names a reach this client has never heard of describes a server this client will not send that query to — silence is never a claim, in either direction, and an unrecognized rung places nowhere on a ladder so it can only be read as its bottom. `normalizeCapabilities()` in `@haverstack/wire-types` applies the rule, so a client reads a foreign discovery response through one function rather than defaulting keys at each use. +**Absent, malformed or unrecognized reads as the least capable value it could stand for.** One rule, for every entry: `'none'` for the reach, `false` for a flag, `0` for a cap, nothing for `sort.fields`, `null` for a limit. A discovery response that omits a group, omits `capabilities` entirely, or names a reach this client has never heard of describes a server this client will not send that query to — silence is never a claim, in either direction, and an unrecognized rung places nowhere on a ladder so it can only be read as its bottom. `normalizeCapabilities()` in `@haverstack/wire-types` applies the rule, so a client reads a foreign discovery response through one function rather than defaulting keys at each use. Two things this does not refuse. A query naming no sort asks for the server's own default order rather than a stated one, so it claims nothing that has to be declared and runs against an empty `sort.fields`. And a `null` limit is permissive by design: it says this client cannot pre-check, not that nothing is enforced — the server's own ceiling still answers with a `413`. diff --git a/docs/spec/apps.md b/docs/spec/apps.md index d9c220c5..18e02168 100644 --- a/docs/spec/apps.md +++ b/docs/spec/apps.md @@ -20,7 +20,7 @@ const plan = await stack.planInstall(manifest, { did: appDid }); await stack.installApp(plan); ``` -Migration functions are not part of a manifest. They are app code, passed to `Stack.open()` on every startup (see [Data model § Type migrations](./data-model.md#type-migrations)), and the app runs them itself — see [Migrating an installed app's types](#migrating-an-installed-apps-types). +Migrations are not part of a manifest. They are app code, passed to `Stack.open()` on every startup (see [Data model § Type migrations](./data-model.md#type-migrations)), and the app runs them itself — see [Migrating an installed app's types](#migrating-an-installed-apps-types). `planInstall()`, `installApp()` and `uninstallApp()` live on `Stack` and are absent from `StackClient`, like `grantType()` and `defineType()`: everything they write is the owner's to write. An app reaches them only through the request it can make over the wire — see [Over the wire](#over-the-wire). @@ -102,7 +102,7 @@ It refuses what no approval could make valid: a type outside the app's own names ## Migrating an installed app's types -[Migration is owner-acting-alone](./access-control.md#what-a-grant-covers), and an installed app's migration functions are its own code, which the owner should not have to run with owner authority. An install closes the gap: a contained app may commit migrations within its own families. `ScopedStack.commitMigration()` — and so `migrateAll()` run by the app over `APIAdapter`, which commits through `POST /records/:id/migrate` — admits a request that is not the owner acting alone when **all** of these hold: +[Migration is owner-acting-alone](./access-control.md#what-a-grant-covers), and an installed app's migrations are its own code, which the owner should not have to run with owner authority. An install closes the gap: a contained app may commit migrations within its own families. `ScopedStack.commitMigration()` — and so `migrateAll()` run by the app over `APIAdapter`, which commits through `POST /records/:id/migrate` — admits a request that is not the owner acting alone when **all** of these hold: 1. The source and target families are both in the namespace of one live install, and both in its `defines`. 2. The target TypeId is in that install's `defines` — a version the owner approved. diff --git a/docs/spec/data-model.md b/docs/spec/data-model.md index af8e6df9..c7d6f50e 100644 --- a/docs/spec/data-model.md +++ b/docs/spec/data-model.md @@ -370,16 +370,16 @@ Reserved, library-defined types: `_config@1` ([Stack initialization](../spec.md# A Type's defining app is the only serious writer of its own types, so migration is **explicit and owner-driven**, not a side effect of a read or an unrelated write. Disk state changes version only via a deliberate `migrateAll()` pass or a per-record `commitMigration()` call — the invariant is that a Record's `typeId` never moves except through one of these two, so `query({ filter: { typeId } })` and grants targeting a type never silently miss not-yet-migrated records. -An app builds one migration per step between adjacent Type versions with `migration(from, to, fn)` and passes them all to `Stack.open()`. The library composes them into a full migration graph, so an app that only knows about v3 doesn't need to know that v1 ever existed. +An app builds one migration per step between adjacent Type versions with `migration(from, to, steps)` and passes them all to `Stack.open()`. The library composes them into a full migration graph, so an app that only knows about v3 doesn't need to know that v1 ever existed. ```ts -const noteV1ToV2 = migration(NoteV1, NoteV2, (content) => ({ ...content, title: '' })); -const noteV2ToV3 = migration(NoteV2, NoteV3, (content) => ({ ...content, pinned: false })); +const noteV1ToV2 = migration(NoteV1, NoteV2, [add('title', '')]); +const noteV2ToV3 = migration(NoteV2, NoteV3, [rename('body', 'text'), add('pinned', false)]); const stack = await Stack.open(adapter, { migrations: [noteV1ToV2, noteV2ToV3] }); ``` -`migration()` takes two [type handles](#type-handles), so `fn` is checked as `StoredContentOf` the first to `StoredContentOf` the second: a step that drops a required field does not compile. A stored enum may hold a value the source handle does not list, so the step carries it through rather than losing it, and a lookup keyed on only the listed values does not compile. A value the target does not accept is refused by its runtime validation when `migrateAll()` writes it. Within a family it refuses a `to` whose `migratesFrom` is not `from.id`, so a step cannot be paired with the wrong handle. +`migration()` takes two [type handles](#type-handles) and checks its steps against both schemas when it runs (see [Migration steps](#migration-steps)). Within a family it refuses a `to` whose `migratesFrom` is not `from.id`, so a step cannot be paired with the wrong handle. **Lineage stays inside a family; a migration can leave it.** `migratesFrom` names an earlier version of the same family, and `typeHandle()` and `defineType()` refuse anything else with `StackBadRequestError`. A step whose `from` and `to` are in different families has no lineage to check, so `migration()` builds it from the two handles alone. This is how an app's own type moves into a shared one — a [commons](../commons/README.md) type, say — without the shared handle naming every family that has ever migrated into it. A migration is a separate value from either handle: a handle is data a manifest carries over the wire, and may be shared by apps that do not run its family's migrations. @@ -391,20 +391,47 @@ The migration registry is **per-stack-instance** and lives in memory — differe - **`presentAt: 'latest'`** — an explicit opt-in on both `get()` and `query()` that applies the registered migration chain in memory before returning. Nothing is written to disk; this is a read-time convenience, never a persistence mechanism. It is a property of the app instance that registered the chain, so it never travels: a server [rejects a request carrying `presentAt`](./wire-format.md#records) rather than dropping it. Throws `StackMigrationError` when a matched Record's version can't be reconciled with what this app instance has registered (see stale-writer behavior below). - **A content patch never migrates.** `mutate()` validates the merged content against the Record's _own current_ stored Type — never the latest — and writes back at the same `typeId`. An unrelated content edit can never fold an invisible schema rewrite into the same version-history entry. This is also why content read through `presentAt: 'latest'` is not writable back wholesale, and why [the content key is a patch](#mutations). A [collection](#type-handles)'s content write to a Record at an older version commits the migration as a write of its own first, then applies the patch. - **Path composition** — migrations between adjacent versions are automatically chained (v1→v2→v3), so apps only ever register one step at a time. -- **`migrateAll("com.example.myapp/note")`** eagerly commits all pending migrations for a type family in one deliberate pass, taking each Record to the end of the registered chain rather than to a version the caller names — which is why it takes a `BaseId` and refuses a versioned `TypeId` — call it at app startup, or after a schema change. It sweeps soft-deleted and unlisted Records (`includeDeleted`/`includeUnlisted` are not caller options in either direction — see [Deletion](./versioning.md#deletion) and [Unlisted records](./unlisted.md)); the one narrower sweep is `{ sweep: 'listed' }`, for an installed app that can reach neither (see [App installs § Migrating an installed app's types](./apps.md#migrating-an-installed-apps-types)). It validates each migrated result against the target Type's schema before writing, and aborts immediately on the first validation failure (a buggy migration function is a bug to surface, not to paper over by skipping the offending records) — anything already committed earlier in the pass stays committed. Previous content is snapshotted to version history before each write. -- **`commitMigration(id, toTypeId, content)`** is the single-record counterpart, changing one Record's `typeId` and `content` together in one step. Unlike `migrateAll()`, `content` here is supplied by the caller rather than produced by a registered `Migration` function — the client-side app that owns `toTypeId` computes it, and the library validates it against `toTypeId`'s schema exactly as `create()`/`mutate()` validate against a schema. This is what backs the wire's `POST /records/:id/migrate` (see [Wire format](./wire-format.md#records)). Under `ScopedStack` it is **owner-acting-alone**, matching `migrateAll()`'s own absence from `StackClient` — no grant or record-level `write` substitutes for it (see [Access control](./access-control.md#type-level-grants)) — except for an installed app within its own families (see [App installs](./apps.md#migrating-an-installed-apps-types)). Previous content and `typeId` are snapshotted to version history first, same as `migrateAll()`. +- **`migrateAll("com.example.myapp/note")`** eagerly commits all pending migrations for a type family in one deliberate pass, taking each Record to the end of the registered chain rather than to a version the caller names — which is why it takes a `BaseId` and refuses a versioned `TypeId` — call it at app startup, or after a schema change. It sweeps soft-deleted and unlisted Records (`includeDeleted`/`includeUnlisted` are not caller options in either direction — see [Deletion](./versioning.md#deletion) and [Unlisted records](./unlisted.md)); the one narrower sweep is `{ sweep: 'listed' }`, for an installed app that can reach neither (see [App installs § Migrating an installed app's types](./apps.md#migrating-an-installed-apps-types)). It validates each migrated result against the target Type's schema before writing, and aborts immediately on the first validation failure (a buggy migration is a bug to surface, not to paper over by skipping the offending records) — anything already committed earlier in the pass stays committed. Previous content is snapshotted to version history before each write. +- **`commitMigration(id, toTypeId, content)`** is the single-record counterpart, changing one Record's `typeId` and `content` together in one step. Unlike `migrateAll()`, `content` here is supplied by the caller rather than produced by a registered migration — the client-side app that owns `toTypeId` computes it, and the library validates it against `toTypeId`'s schema exactly as `create()`/`mutate()` validate against a schema. This is what backs the wire's `POST /records/:id/migrate` (see [Wire format](./wire-format.md#records)). Under `ScopedStack` it is **owner-acting-alone**, matching `migrateAll()`'s own absence from `StackClient` — no grant or record-level `write` substitutes for it (see [Access control](./access-control.md#type-level-grants)) — except for an installed app within its own families (see [App installs](./apps.md#migrating-an-installed-apps-types)). Previous content and `typeId` are snapshotted to version history first, same as `migrateAll()`. Because `content` is a full replacement written under a new `typeId`, a migration commit is create-shaped at the destination and update-shaped over the Record as it stands, and owes both sets of integrity checks. DID bindings are held to immutability across the union of the two families' binding fields — a card can neither shed its `did` by migrating out of `_entity`/`_app` nor pick one up on the way in — and to uniqueness in the destination family (see [Identity § DID bindings](./identity.md#did-bindings)). An `_attachment@1` Record's `fileId`, `mimeType` and `size` stay immutable, and a Record arriving from outside that family is held to the same mimeType-establishment check `create()` applies. Migrating _into_ `_group` is refused outright: a group's `admin` roster entry is stamped at creation and a migration cannot stamp one, so it would produce a group nobody but the owner can manage — version-to-version migration within `_group` stays open and carries the existing roster with it. - **`migrateAll()` applies these same checks**, on the same shared write path. That a `Migration` function is app code rather than a request body is not a trust boundary here: the app calling `commitMigration()` is the same app that passed the function to `Stack.open()`, and neither is entitled to move a DID binding or repoint an attachment. A path can itself [cross type families](#type-migrations), which is precisely what these checks are about. A migration function that would violate one aborts the pass like any other validation failure. + **`migrateAll()` applies these same checks**, on the same shared write path. That a migration is app code rather than a request body is not a trust boundary here: the app calling `commitMigration()` is the same app that passed the migration to `Stack.open()`, and neither is entitled to move a DID binding or repoint an attachment. A path can itself [cross type families](#type-migrations), which is precisely what these checks are about. A migration that would violate one aborts the pass like any other validation failure. **Stale-writer behavior.** A Record whose version this app instance can't reconcile — older than what it was passed _and_ not bridged by a migration path, or newer than anything it has ever `defineType()`'d — is an explicit error (`StackMigrationError`) under `presentAt: 'latest'`, not a silent pass-through. This covers both directions of "the same app at two versions" meeting via a shared stack. Reading the Record as stored (the default, no `presentAt`) always succeeds regardless — the stale-writer signal only fires when the app explicitly asks for the migrated view and the library can't honestly provide one. +#### Migration steps + +A migration is a list of steps, applied in order to a copy of the stored content. A field no step names is carried over as it is. The vocabulary is named after what Rails, Prisma, Avro and Cambria already call these operations: + +| Step | Effect | +| ------------------- | -------------------------------------------------------------------------------------------------------------- | +| `rename(from, to)` | moves a field's value to a new name | +| `map(field, table)` | replaces each value `table` lists with its mapping and keeps any other; an array is mapped element-wise | +| `add(field, value)` | sets a field to a constant | +| `drop(field)` | removes a field | +| `derive(field, fn)` | sets a field to `fn(content)`, where `content` is the record as stored at `from`; `undefined` leaves it absent | + +Steps name top-level fields. A nested change is a `derive` of the field that holds it. + +**Steps are declared rather than written as a function so that core can read them.** Each says what became of a field, which is what lets a [collection](#type-handles) carry a content filter or sort across versions (see [Content queries across versions](#content-queries-across-versions)). `derive` is the one opaque step. It covers what the others cannot — a change of kind, a split, a value computed from other fields — and a field it produces does not translate. A stored enum may hold a value the source handle does not list, so `map` keeps an unlisted value rather than losing it, and `derive`'s function is typed on `StoredContentOf` the source handle. + +**`migration()` checks the steps against both schemas** and throws `StackMigrationError` naming the step or field at fault: + +- a `rename`, `map` or `drop` of a field that is not there at that point in the list, or a `rename` or `add` onto one that is; +- a field left at the end that `to` does not declare — drop or rename it; +- a field `to` requires that no step produces and `from` does not carry; +- a carried field — unchanged, renamed or mapped — that `to` declares in a way its stored values could fail: a different kind (an `enum`, `string` or `text` into `string` or `text` aside), optional into required, an open container into a declared one, or an enum value, after any `map`, that `to`'s enum does not list. Containers are compared field by field; +- a `map` of a field that is not an `enum`, `string` or `text`, or an array of one; +- an `add` whose value `to`'s declaration of the field refuses. + +What `derive` returns is known only when it runs, so validation against the target Type when `migrateAll()` or `commitMigration()` writes it stays the backstop. + ### Type handles A schema written once as a literal gives the compiler everything it needs to type content. `typeHandle({ id, name, schema, migratesFrom })` returns a plain, frozen value carrying the fields `defineType()` takes — the `TypeId` (`id`), the display `name`, the `schema` and, for a version after the first, `migratesFrom` — plus its family (`baseId`). `migratesFrom` may be given as the previous version's handle or its `TypeId`, and is stored as the `TypeId`. `ContentOf` derives the content type a write supplies, and `StoredContentOf` the one a read returns. A required field is present in `ContentOf`; every other field is optional. A [collection](#type-handles) derives the rest from its content type `C`: `StoredContent` is what its reads return, the same as `StoredContentOf`, and `ContentPatch` is the `contentPatch` it takes, which makes every field optional and allows `null` only on a field that is not required, since a required field can be replaced but not removed. -A handle is a `DefineTypeOptions`, so `defineType(handle)` and a manifest's `types` take it as written, and `name` and `migratesFrom` are written once. It carries no migration function, which keeps it data: see [Type migrations](#type-migrations). +A handle is a `DefineTypeOptions`, so `defineType(handle)` and a manifest's `types` take it as written, and `name` and `migratesFrom` are written once. It carries no migration, which keeps it data: see [Type migrations](#type-migrations). A handle names exactly one version. A call that means the whole family takes `handle.baseId`, never `handle.id`. @@ -422,11 +449,11 @@ A read types an `enum` field as its listed values or any other string (`'want' | - `get()` is a read the app asked for by ID, so it throws `MisfitError`, carrying the `reason` and the record as stored. - `subscribe()` delivers a change whose record does not fit with `misfit` in place of `record`. -A record at an older version that no passed migration carries to the handle's is not a misfit: that is the app's setup, and a read of it throws `StackMigrationError`, as it throws whatever a failing migration function throws. A subscription cannot throw at its emitter, so such a change arrives without its `record`, as it does wherever the emitter cannot supply one, and `get()` says why. `MisfitError` is raised only in the app's process and has no wire representation, so it sits [outside the taxonomy](./wire-format.md#the-taxonomy-root). +A record at an older version that no passed migration carries to the handle's is not a misfit: that is the app's setup, and a read of it throws `StackMigrationError`, as it throws whatever a failing `derive` throws. A subscription cannot throw at its emitter, so such a change arrives without its `record`, as it does wherever the emitter cannot supply one, and `get()` says why. `MisfitError` is raised only in the app's process and has no wire representation, so it sits [outside the taxonomy](./wire-format.md#the-taxonomy-root). **A subscription covers the family.** A change's `record`, when present, is migrated to the handle's version; the change's own `typeId` stays the stored one, as `RecordChange.typeId` documents. A change whose record has `deletedAt` set carries neither `record` nor `misfit`, on either surface, and neither does a change that [migrates the record out](#type-migrations) of the family, which the filter matches by its previous `typeId`. The change filter takes every key but `typeId` and `baseId`. -**A content filter or sort reaches only records stored at the handle's version.** The adapter matches and orders content as stored, before any record is migrated, and a migration is an opaque function, so a predicate written in the handle's shape cannot be translated into an older version's. Run across the family, a field a migration renamed, derived or remapped would silently miss the older records or misplace them. So a `query()` whose `filter.content` or `filter.contentPresent` is non-empty, or whose sort is a `contentField`, reads only records whose `typeId` is the handle's, even for a field no migration touched, since core cannot tell which fields those are. The untyped `query({ filter: { baseId, content } })` still matches across the family, returning records as stored. The first page of such a query sets `skipped.older` or `skipped.newer` when records at an older or newer version match the rest of the query: the results may then be missing records the filter was meant to find. A later page carries no `skipped`, since the flags depend on the query, not the page. Each is one `limit: 1` read, run beside the first page's own, and a scoped read that comes back empty with a cursor still set counts as a match rather than reading on, so a flag can be set when nothing was skipped but is never clear when something was. Mixed versions are a lasting state, not a backlog, since an app at an older version may keep writing its own, so the app decides what `skipped.older` means to it; `migrateAll()` clears it only until the next such write. `skipped.newer` says the app is behind, which an unfiltered page reports as misfits; a narrowed page carries none. A query with no content keys reads the whole family. The keys are typed against the handle's version: `filter.content` and `filter.contentPresent` take the content's paths (`ContentPathOf`), through declared objects and arrays, with any suffix below an `open` object or array, and a value of the field's type as a write takes it (an array's element type, since matching is element-wise) or `null`. A path reaching an object takes only `null`, since an adapter compares the value a path reaches and an object never equals one; a value below an `open` node is any scalar. Typing values as writes take them means an enum value the handle does not list is filtered for with a cast, the price of refusing a misspelled one; `sort.contentField` takes only a top-level scalar (`SortableFieldOf`). Path typing stops six segments deep and accepts any suffix below that, a limit of the compiler, not of the [32-segment path](#nested-content-paths). `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. +**A content filter or sort spans the versions the family's migrations can carry it to**, and reaches no further: see [Content queries across versions](#content-queries-across-versions). The keys are typed against the handle's version: `filter.content` and `filter.contentPresent` take the content's paths (`ContentPathOf`), through declared objects and arrays, with any suffix below an `open` object or array, and a value of the field's type as a write takes it (an array's element type, since matching is element-wise) or `null`. A path reaching an object takes only `null`, since an adapter compares the value a path reaches and an object never equals one; a value below an `open` node is any scalar. Typing values as writes take them means an enum value the handle does not list is filtered for with a cast, the price of refusing a misspelled one; `sort.contentField` takes only a top-level scalar (`SortableFieldOf`). Path typing stops six segments deep and accepts any suffix below that, a limit of the compiler, not of the [32-segment path](#nested-content-paths). `query()` takes neither `typeId` nor `presentAt`, which would change which records a page holds or the shape they arrive in, and drops either if an untyped caller passes it; `subscribe()` drops a filter's `typeId` the same way. **Every verb that returns a record returns it as `get()` would.** The write already returned the record, so narrowing it costs no round trip, and a `restoreVersion()` to a pre-migration snapshot reads back migrated, like any older record. A write is checked against the read it already makes, before anything lands: a record outside the family is refused with `StackNotFoundError`, the answer `get()` gives, and so is a `restoreVersion()` to a snapshot taken before the record migrated into the family, which is not a version of a record in the collection. A content write to a record stored at a newer version is refused with `MisfitError` before anything is written, since its patch would be validated against a schema the handle does not know. Any other write that leaves a misfit lands, then throws `MisfitError` carrying the stored record, so the app knows the write landed. A deleted record, or a stale `ifVersion`, is refused first, as the untyped write refuses it. @@ -434,6 +461,24 @@ A record at an older version that no passed migration carries to the handle's is The derived types are a convenience over [runtime validation](#types), which stays the guarantee. A Type known only at runtime has no static shape and stays `Record`. +#### Content queries across versions + +**A collection rewrites a content filter or sort into each older version's shape.** The adapter matches and orders content as stored, before any record is migrated, so a predicate written in the handle's shape would silently miss a record whose field a migration renamed or remapped, and misplace it in a sort. Core composes the [steps](#migration-steps) between each older version and the handle into what became of each of the handle's top-level fields — a field of that version, its values possibly remapped; a constant; or something derived — and rewrites the query into one arm per version: [`filter.contentByType`](#content-by-type) for a filter and `sort.contentFieldByType` for a sort. Each key translates by its top-level field, and the rest of its path is kept, since steps reach top-level fields only: + +| Step on the path | Filter | Sort | +| ---------------- | -------------------------------------------------------------------------------------------- | ----------------------------------------------------- | +| `rename` | translated: the path renamed in the older arm | translated: the field per typeId | +| `map` | translated: the value replaced by the older value that maps to it | **narrowed** — remapped values don't keep their order | +| `add` | decided in core: the constant is matched against the key, which keeps or drops the older arm | translated: the constant per typeId | +| `drop` | nothing to translate: the handle does not declare the field | nothing to translate | +| `derive` | narrowed | narrowed | + +A field the handle declares optional and no step produces is the constant "no value". A `map` keeps a value it does not list, so more than one older value can arrive as the one a key asks for: `map('status', { done: 'finished' })` carries an unlisted `finished` through as well as remapping `done`. An arm matches one value per path, so it asks for a mapped value first, and each other source becomes a residual the first page probes for, as it probes a narrowed version: a match sets `skipped.older`. One such key per arm is split this way; a second narrows the version. A key no older value maps to drops the older arm, as a constant that does not match does. A key on a field the handle does not declare, which only an untyped caller can write, is narrowed. **A narrowed version is left out of the query**; a dropped arm is left out too, but only because no record there can match. Newer versions stay out of reach either way: the handle cannot represent them. + +Records at an older version arrive migrated, as on an unfiltered page, and a sort orders them as their migrated values would order, since a carried field keeps its kind. When no older version is left in, the query is sent as a plain `content` filter or `contentField` sort on the handle's `typeId`, which every adapter reaching content answers. An adapter declaring [`filter.contentByType`](./adapters.md#adapter-capabilities) `0` — or `sort.contentFieldByType` `0`, for a content sort — gets that query, with every older version narrowed. One declaring a lower cap than the family has versions to query gets the newest versions that fit, and the rest are narrowed. + +**`skipped` says which side the query could not reach.** The first page sets `skipped.older` when records at a narrowed older version match the rest of the query, and `skipped.newer` when records at a newer version do: the results may then be missing records the filter was meant to find. A version is reported only when the filter or sort touches a field that does not translate across some step between it and the handle, or when a residual of its arm matches — a dropped arm is not a skip. A later page carries no `skipped`, since the flags depend on the query, not the page. Each is one `limit: 1` read, run beside the first page's own, and a scoped read that comes back empty with a cursor still set counts as a match rather than reading on, so a flag can be set when nothing was skipped but is never clear when something was. Mixed versions are a lasting state, not a backlog, since an app at an older version may keep writing its own, so the app decides what `skipped.older` means to it; `migrateAll()` clears it only until the next such write. `skipped.newer` says the app is behind, which an unfiltered page reports as misfits; a content query's page carries none. A query with no content keys reads the whole family. The untyped `query({ filter: { baseId, content } })` still matches across the family, returning records as stored. + ### Additive evolution within a version Not every schema change needs a version bump. **Additive-in-place** changes — new optional fields only — can be added to a Type's schema without minting a new version, and are the default path for evolving a type family: @@ -455,8 +500,6 @@ A schema accumulating many optional fields is a named smell that a consolidating The boundary between "accept in place" and "bump the version" is exactly what `defineType()`'s [schema drift detection](#schema-drift-detection) mechanically enforces — a diff, not the hash alone, since the hash necessarily changes on any legal additive diff too. -> **Not yet implemented:** validation of migration function output against the target schema at _registration_ time (write-time validation, in `migrateAll()`, is the enforced backstop today). - ### Undeclared content fields **A content field the Record's Type does not declare is rejected with `StackValidationError` (422)** — on `create()`, in a content patch, and on `commitMigration()` against the destination Type — naming the field and the path it sits at. The schema is the Record's shape, and it holds at every depth: an undeclared key inside a declared `object`, or inside an `object` an `array` declares as its items, is refused with its full path (`address.postcode`, `emails[1].label`). @@ -537,6 +580,7 @@ type RecordFilter = { // Content fields (exact match; the key is a dot-separated path) content?: { [key: string]: unknown }; contentPresent?: string[]; // paths that must hold a value + contentByType?: { [typeId: TypeId]: ContentArm }; // one arm per Type; see Content by type // Full-text search (capability varies by adapter) search?: string; @@ -550,6 +594,11 @@ type DateRange = { before?: Date; after?: Date; }; + +type ContentArm = { + content?: { [key: string]: unknown }; + contentPresent?: string[]; +}; ``` **`relatedTo` names a label, a target, or both — never neither**, and a filter naming neither is rejected with `StackBadRequestError` rather than widening to every Record carrying a relationship. Its target follows the same naming rules as a stored one (above), checked the same way. A `RelationshipTargetPattern` is the association's own target shape with the parts a query may leave open. Each half is a pattern: a bare `label` matches every target under it; an `external` target with no `id` matches the whole namespace, which is how a syndication tool asks what it has already published. A `record` target with no `stackUrl` matches only local targets — absence names this stack rather than acting as a wildcard, so a Record referenced in someone else's stack is reachable only by naming that stack. Matching is exact within a target kind and never across kinds. @@ -594,6 +643,19 @@ By default, `query()` (like `get()`) returns Records exactly as stored — see [ **`query()` never returns the `_config` record**, regardless of filter — it's addressable only by ID, via `get('_config')` or the adapter's own typed `ownerEntityId`/`timezone` properties (see [Stack initialization](../spec.md#stack-initialization)). This is the one exception to "adapters are storage engines, `Stack` is the invariant layer": the exclusion must live in the adapter's own query predicate (a `WHERE` clause, or the equivalent for an in-memory adapter) rather than be post-filtered by `Stack`, since post-filtering after the adapter applies `limit` would silently under-fill a page. Every adapter — including test doubles — implements this exclusion directly; it is not optional convention. +#### Content by type + +**`contentByType` is a content predicate keyed by `TypeId`.** A Record matches when its `typeId` is one of the keys and its content satisfies that key's arm, whose `content` and `contentPresent` mean what they mean at the top of the filter. It is how a [collection](#content-queries-across-versions) carries a content filter across a family's versions: core generates it from the migration chain, and callers don't write it. + +**It splits a query along Types; it is not a disjunction**, and the filter stays a flat AND: + +- **Every arm is pinned to a `typeId`**, and a filter carrying `contentByType` names at least one. An empty map is refused with `StackBadRequestError` (400), so the key cannot encode to a filter that matches everything. +- **An arm is flat**: `content` and `contentPresent`, nothing else, and no nesting. An arm naming neither matches every Record at its `typeId`, which is what `typeId` alone matches. +- **It never sits beside a top-level `content` or `contentPresent`**: a filter with both is refused with 400, since which one applies to a Record would be ambiguous. Every other key applies to every arm, and a `typeId` beside it intersects with its keys. +- **It is bounded** by the adapter's `filter.contentByType`, the most arms it takes; a filter with more is refused with 400 naming that capability (see [Capability-gated filters](#capability-gated-filters)). + +It discloses nothing new: each arm is a query the caller could already run, its `typeId` with its content filter, and the permission check applies per Record as for any query. Types partition Records, so no Record is matched by two arms, and the cost of _n_ arms stays within what _n_ queries would cost — what [Wire format § Bounding query cost](./wire-format.md#bounding-query-cost) already leaves to the server. **It is not a first step toward general OR.** A disjunction across Types is the only one a migration chain needs, and match-any forms stay refused for the reason [above](#filter): a filter that can encode to nothing can silently widen a query when it crosses the wire. + ### Sorting and pagination ```ts @@ -601,12 +663,15 @@ type StackQuery = { filter?: RecordFilter; sort?: | { field: 'createdAt' | 'updatedAt' | 'version'; direction?: 'asc' | 'desc' } - | { contentField: string; direction?: 'asc' | 'desc' }; // see Sorting by a content field + | { contentField: string; direction?: 'asc' | 'desc' } // see Sorting by a content field + | { contentFieldByType: { [typeId: TypeId]: SortKey }; direction?: 'asc' | 'desc' }; limit?: number; cursor?: string; // Opaque cursor for page-based pagination presentAt?: 'stored' | 'latest'; // see Type migrations }; +type SortKey = { field: string } | { value: string | number | null }; + type QueryResult = { records: StackRecord[]; cursor: string | null; // null = no further pages @@ -628,7 +693,7 @@ Pagination is cursor-based rather than offset-based, so it works consistently ac | `getAttachmentRecords()` | first-recorded order, oldest first ([Attachments](./attachments.md)) | | `listTypeGrants()` | newest first | -**A sort names either a native column or a content field, never both**, and a request naming both is rejected with `StackBadRequestError` (**400**) rather than resolved in one direction. They are two members rather than one widened `field` because a content field may be named `version`, `createdAt` or `updatedAt`, and a `'content.publishedAt'` prefix would collide with the [path separator](#filter) a filter key is split on. +**A sort names one of a native column, a content field or a content field per Type**, and a request naming more than one is rejected with `StackBadRequestError` (**400**) rather than resolved in one direction. They are two members rather than one widened `field` because a content field may be named `version`, `createdAt` or `updatedAt`, and a `'content.publishedAt'` prefix would collide with the [path separator](#filter) a filter key is split on. **`sort.field` and `sort.direction` are validated, not merely typed.** The `'asc' | 'desc'` and three-field types are a compile-time contract only; a request arriving over the wire (a server mapping `?sort=`/`?direction=`) or from a delegated app supplies raw strings. A SQLite record adapter interpolates the direction into its `ORDER BY`, so `Stack.query()`/`ScopedStack.query()` reject a field or direction outside the enumerated set with `StackBadRequestError` (**400**, `bad_request`) before it reaches an adapter — the same posture as a malformed cursor. The value never reaches SQL as anything but one of the two keywords. @@ -656,6 +721,8 @@ A caller that genuinely needs a count follows `cursor` to exhaustion and counts **Where a query spans Types that declare one field name differently, numbers sort before text**, and `direction` reverses that with everything else. Stating it here rather than leaving it to each engine's collation is what stops two adapters answering one query in two orders. +**`sort.contentFieldByType` names the sort value per Type**, as [`filter.contentByType`](#content-by-type) names a filter per Type, and is how a [collection](#content-queries-across-versions) orders a family across its versions. A Record whose `typeId` is a key orders by that key's `field`, read as its own Type declares it, or by the constant `value`: a number orders as a number, a string as text, and `null` as no value. A Record whose `typeId` is not a key holds no value. The constant arrives already reduced, so an adapter orders it without knowing what kind it stands for — a `date` constant travels as the epoch milliseconds a stored `date` orders by. Each `field` is one segment, as `contentField` is; the map names at least one key and at most the adapter's `sort.contentFieldByType`, and anything else is refused with 400. A cursor resumes only the map it was minted under. + **A content sort orders the stored shape, not the migrated one.** Migrations run in memory after the query returns (see [Type migrations](#type-migrations)), so a Stack mid-migration, sorting on a field a migration rewrites, gets an order derived from what is on disk. The distinction cannot arise for the native three. Sorting by an association — a relationship target, or a tag — is a different mechanism with a different cost and is not expressible; `order` on an association is ruled out by the [commons convention](../commons/README.md) that associations point and mark but never carry data. @@ -680,10 +747,12 @@ Locale-correct ordering is out of reach for a stored key regardless of effort, b `content` filtering and `search` depend on adapter capabilities (see [Adapter capabilities](./adapters.md#adapter-capabilities)). `query()` fails loud rather than silently widening: `Stack.query()` checks `filter.content`/`filter.search` against `adapter.capabilities` before dispatching, and throws `StackBadRequestError` (`bad_request`/400) if the adapter hasn't declared the matching capability — local and remote behave identically. -**A refusal names the capability it was refused for.** `StackBadRequestError` carries a `capability` field holding the path into [adapter capabilities](./adapters.md#adapter-capabilities) the query needed — `'filter.content'`, `'filter.contentPresent'`, `'filter.search'`, `'sort.fields'` or `'sort.contentField'`. A client presenting the refusal in its own vocabulary reads the name from there rather than re-deriving it from the query, which is how one rule becomes two answers that can disagree; `adapter-api` raises `APIAdapterCapabilityError` carrying the same name. A `StackBadRequestError` thrown for the query's own shape — a malformed content path, a sort naming both a native field and a content field — carries none, because that is the caller's error at every capability level and naming a capability would point at a discovery entry that is not the problem. The field is local only: the [wire error body](./wire-format.md#error-responses) carries `bad_request` and a message, as it does for every other `StackBadRequestError`. +**A refusal names the capability it was refused for.** `StackBadRequestError` carries a `capability` field holding the path into [adapter capabilities](./adapters.md#adapter-capabilities) the query needed — `'filter.content'`, `'filter.contentPresent'`, `'filter.contentByType'`, `'filter.search'`, `'sort.fields'`, `'sort.contentField'` or `'sort.contentFieldByType'`. A client presenting the refusal in its own vocabulary reads the name from there rather than re-deriving it from the query, which is how one rule becomes two answers that can disagree; `adapter-api` raises `APIAdapterCapabilityError` carrying the same name. A `StackBadRequestError` thrown for the query's own shape — a malformed content path, a sort naming both a native field and a content field — carries none, because that is the caller's error at every capability level and naming a capability would point at a discovery entry that is not the problem. The field is local only: the [wire error body](./wire-format.md#error-responses) carries `bad_request` and a message, as it does for every other `StackBadRequestError`. **`contentPresent` needs `filter.contentPresent` on top of a content reach**, a separate flag on the same reasoning the reach ladder rests on: a server promising to match a content value has not thereby promised to answer whether one is there, and reading it as such would hand a client the unfiltered superset that ignoring the filter produces. A multi-segment presence path needs the `'path'` rung too, exactly as a filter key does. +**`contentByType` and `contentFieldByType` need their own capabilities**, `filter.contentByType` and `sort.contentFieldByType`: each is the most keys the adapter takes, and `0` declines the shape. An arm's keys are gated as the same keys at the top of a filter are, so a multi-segment path in an arm needs `'path'`, and an arm's `contentPresent` needs `filter.contentPresent`. + **A sort is gated the same way.** `sort.contentField` needs the adapter's `sort.contentField`, and a native `sort.field` must appear in its `sort.fields`; a sort an adapter hasn't declared throws `StackBadRequestError` rather than being answered in some other order — which a caller reading one bounded page has no way to notice. `sort.contentField` and `filter.content` are independent: a server may order by a content field without offering to filter on one, or the reverse. A `search` that sanitizes to nothing (a bare `*`, punctuation-only input) is treated as a legitimate zero-match query rather than an omitted filter — matching nothing is honest; silently returning the full table is not. diff --git a/docs/spec/versioning.md b/docs/spec/versioning.md index 132023bc..0d124747 100644 --- a/docs/spec/versioning.md +++ b/docs/spec/versioning.md @@ -74,7 +74,7 @@ A Record's sharing history is the [journal's](./journal.md#reading-it) to tell, **Snapshot content is validated against the type it claims** (`typeId` as stored in the snapshot), not the Record's current type — a snapshot taken before a migration is `@1`-shaped, and validating it against a since-migrated `@2` schema would wrongly reject a legitimate restore. A snapshot that fails validation against its own claimed type — in-place schema drift, or a corrupted/buggy adapter — throws `StackValidationError` instead of being written back; restore is a recovery path, not a backdoor around content validation. -Restoring a pre-migration snapshot therefore also restores its old `typeId`, leaving the Record legitimately **stale** rather than mislabeled. No forward-migration happens at restore time — migration functions are app code (see [Type migrations](./data-model.md#type-migrations)), so restore behaves the same locally as through the server-side restore endpoint, which cannot run them either. A stale restored Record self-heals the same way any other stale Record does: on the owning app's next `migrateAll()` sweep. +Restoring a pre-migration snapshot therefore also restores its old `typeId`, leaving the Record legitimately **stale** rather than mislabeled. No forward-migration happens at restore time — migrations are app code (see [Type migrations](./data-model.md#type-migrations)), so restore behaves the same locally as through the server-side restore endpoint, which cannot run them either. A stale restored Record self-heals the same way any other stale Record does: on the owning app's next `migrateAll()` sweep. **A restore never moves the Record.** Containment is [the journal's to keep](#version-history), not the snapshot's, so a restore settles nothing about it: the Record comes back in whatever container it is in now, whichever one it sat in when the snapshot was taken. Undoing a move is a change set's `parentId`, reading the entry's `previousParentId` for where it came from. That is what keeps a restore off every containment rule: it adds no edge, so there is no ancestor chain to walk and no destination to gate, and a container purged since the snapshot was taken costs the Record nothing. **Listing state is settled the same way — which is to say not at all.** A restore never lists or unlists a Record; `unlist`/`list` is a change set's `unlisted`. diff --git a/docs/spec/wire-format.md b/docs/spec/wire-format.md index e2c5f625..0d0e1f8b 100644 --- a/docs/spec/wire-format.md +++ b/docs/spec/wire-format.md @@ -16,8 +16,12 @@ GET /.well-known/stack "entityId": "did:key:z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK", "timezone": "America/New_York", "capabilities": { - "filter": { "content": "path", "contentPresent": true, "search": true }, - "sort": { "fields": ["createdAt", "updatedAt", "version"], "contentField": true }, + "filter": { "content": "path", "contentPresent": true, "contentByType": 16, "search": true }, + "sort": { + "fields": ["createdAt", "updatedAt", "version"], + "contentField": true, + "contentFieldByType": 16 + }, "limits": { "attachmentBytes": 52428800, "contentBytes": 1048576 } }, "auth": { "methods": ["did-challenge"] }, @@ -321,7 +325,9 @@ This is what lets a client report a mutation's outcome without a second read, an **`filter.contentPresent` travels in the `POST /records/query` body only**, like `filter.content` itself — there is no `GET /records` spelling, since a server exposing that endpoint alone declares `filter.content: "none"` and offers neither. A server declaring `filter.contentPresent: false` MUST refuse the filter with `400` rather than answer without it. -**A sort names a native column or a content field, and the parameter it arrives in says which** — the same shape the relationship filter uses above, and the only way to keep a content field named `version` distinct from the native column on a raw query string. A request carrying both `sort` and `sortContent` MUST be rejected with `400` rather than resolved in one direction; in a `POST /records/query` body the same two spellings are `sort.field` and `sort.contentField`. A server declaring `sort.contentField: false` refuses `sortContent` with `400`. What the ordering itself must be — absent values last, numbers before text, and the [folded text key](./data-model.md#text-ordering) a server MUST reproduce — is in [Data model § Sorting by a content field](./data-model.md#sorting-by-a-content-field). +**`filter.contentByType` and `sort.contentFieldByType` travel in the `POST /records/query` body only**, for the reason `filter.contentPresent` does. A server declaring `0` for either MUST refuse it with `400`, and one receiving more keys than it declares MUST refuse the request with `400` rather than answer some of the arms. An empty map, an arm carrying anything but `content` and `contentPresent`, a `contentByType` beside a top-level `content` or `contentPresent`, and a sort key that is neither `{ "field": string }` nor `{ "value": string | number | null }` are `400` as well. See [Data model § Content by type](./data-model.md#content-by-type). + +**A sort names a native column or a content field, and the parameter it arrives in says which** — the same shape the relationship filter uses above, and the only way to keep a content field named `version` distinct from the native column on a raw query string. A request carrying both `sort` and `sortContent` MUST be rejected with `400` rather than resolved in one direction; in a `POST /records/query` body the same two spellings are `sort.field` and `sort.contentField`, and `sort.contentFieldByType` is a third that no request may combine with either. A server declaring `sort.contentField: false` refuses `sortContent` with `400`. What the ordering itself must be — absent values last, numbers before text, and the [folded text key](./data-model.md#text-ordering) a server MUST reproduce — is in [Data model § Sorting by a content field](./data-model.md#sorting-by-a-content-field). **`?sort=` or `?sortContent=` without `?direction=` is ascending, and a request with no sort is `createdAt`, newest first.** In a `POST /records/query` body, `sort.direction` absent means the same. A server built on `Stack.query()` inherits both; see [Data model § Sorting and pagination](./data-model.md#sorting-and-pagination). @@ -329,7 +335,7 @@ This is what lets a client report a mutation's outcome without a second read, an `@haverstack/core/wire` exports a conforming implementation of both — `parseQueryParams()` for the params above and `parseQueryBody()` for the JSON body — the same way [Authentication](#authentication) points at the handshake helpers. The table stays normative for a server written in another language; a server built on core inherits the grammar rather than transcribing it. -**Two `StackQuery` fields have no wire encoding at all, and a server MUST reject a request carrying either with `400` rather than ignoring it.** `baseId` is [resolved client-side against registered Types](./data-model.md#filter) before a request is built, so the wire only ever carries the concrete `typeId` set it resolves to; a server that dropped it would answer a family query with an unfiltered result set. `presentAt` applies registered migration functions, which are [app code rather than server code](#migration-commit), so no server can honor it; a server that dropped it would return records as stored to a caller who asked for them migrated. Both are the silent degradation [capability-gated filters](./data-model.md#capability-gated-filters) are already refused for — a result presented as the requested query but wider, or staler, than what was asked. +**Two `StackQuery` fields have no wire encoding at all, and a server MUST reject a request carrying either with `400` rather than ignoring it.** `baseId` is [resolved client-side against registered Types](./data-model.md#filter) before a request is built, so the wire only ever carries the concrete `typeId` set it resolves to; a server that dropped it would answer a family query with an unfiltered result set. `presentAt` applies registered migrations, which are [app code rather than server code](#migration-commit), so no server can honor it; a server that dropped it would return records as stored to a caller who asked for them migrated. Both are the silent degradation [capability-gated filters](./data-model.md#capability-gated-filters) are already refused for — a result presented as the requested query but wider, or staler, than what was asked. A client MUST refuse both before choosing an encoding, rather than relying on that `400`. Only the JSON body carries the fields far enough to be rejected; the params table above has nowhere to put them, so a client that left the check to the server would answer the same query with an error or an unfiltered result set depending on which endpoint the server's `filter.content` reach selected. `@haverstack/core/wire` exports `assertQueryTravels()` for this, the build-side half of the parsers' refusal; `APIAdapter` applies it to every query. It reads an explicitly `undefined` field as absent, where the parsers judge by key presence — a `StackQuery` is spread and destructured rather than written key by key, and `Stack.query()` already resolves an `undefined` `baseId` to no filter at all. @@ -390,7 +396,7 @@ For `typeId: "_attachment@1"`, a non-owner requester gets `403` regardless of gr ### Migration commit -`POST /records/:id/migrate` is the only way a record's `typeId` changes after creation. Body: `{ "toTypeId": "...", "content": {...} }` — the full post-migration content, computed client-side by the type's owning app (migration functions are app code, not server code) and validated by the server against `toTypeId`'s schema before writing. This is what an app uses to commit a pending migration alongside new content (a change set carries no `typeId`, so `PATCH` cannot), and what `stack.migrateAll()` uses for each record in a batch pass. `Stack.commitMigration()`/`ScopedStack.commitMigration()` is the client-side entry point that backs this endpoint for a single record — see [Type migrations](./data-model.md#type-migrations). A server built on `ScopedStack` serves this endpoint to the **stack owner**, and to an installed app migrating within the families its install claims (see [App installs § Migrating an installed app's types](./apps.md#migrating-an-installed-apps-types)), and answers `403` otherwise: migration is owner-driven, and no grant alone confers it (see [Access control](./access-control.md#type-level-grants)). Like every other endpoint that bumps a record's version, it accepts `If-Match` — a migration commit replaces content wholesale, so it is precisely the write a caller most needs to be able to fence. `stack.migrateAll()` sends none, since a batch pass doesn't know each record's version going in; a single `commitMigration()` passes whatever `ifVersion` its caller supplied. +`POST /records/:id/migrate` is the only way a record's `typeId` changes after creation. Body: `{ "toTypeId": "...", "content": {...} }` — the full post-migration content, computed client-side by the type's owning app (migrations are app code, not server code) and validated by the server against `toTypeId`'s schema before writing. This is what an app uses to commit a pending migration alongside new content (a change set carries no `typeId`, so `PATCH` cannot), and what `stack.migrateAll()` uses for each record in a batch pass. `Stack.commitMigration()`/`ScopedStack.commitMigration()` is the client-side entry point that backs this endpoint for a single record — see [Type migrations](./data-model.md#type-migrations). A server built on `ScopedStack` serves this endpoint to the **stack owner**, and to an installed app migrating within the families its install claims (see [App installs § Migrating an installed app's types](./apps.md#migrating-an-installed-apps-types)), and answers `403` otherwise: migration is owner-driven, and no grant alone confers it (see [Access control](./access-control.md#type-level-grants)). Like every other endpoint that bumps a record's version, it accepts `If-Match` — a migration commit replaces content wholesale, so it is precisely the write a caller most needs to be able to fence. `stack.migrateAll()` sends none, since a batch pass doesn't know each record's version going in; a single `commitMigration()` passes whatever `ifVersion` its caller supplied. ### Response envelope @@ -415,6 +421,8 @@ The record adapters sanitize a `search` string's _complexity_ — wildcards stri The mitigation therefore lives where the engine is driven, not in the sanitizers: run record-adapter queries off the request thread (a worker per connection, or a worker pool), so a slow query can be interrupted — `sqlite3_interrupt`, or terminating the worker — and apply the per-request deadline at that boundary. A server that does none of this is conforming but will serve exactly as well as its slowest query. +A [`filter.contentByType`](./data-model.md#content-by-type) of _n_ arms costs what _n_ content queries would, so it adds nothing to bound but a count, which a server sets by declaring its cap. + `timeout` is a distinct code because a client acts on it differently from every other failure in the table: nothing was applied, and the same request may well succeed if narrowed or retried. Reusing `bad_request` would say the opposite. See [Data model § Capability-gated filters](./data-model.md#capability-gated-filters). ### Request size limits diff --git a/packages/adapter-api/tests/api.test.ts b/packages/adapter-api/tests/api.test.ts index 5302c20f..f4c4275e 100644 --- a/packages/adapter-api/tests/api.test.ts +++ b/packages/adapter-api/tests/api.test.ts @@ -1076,6 +1076,30 @@ describe('queryRecords', () => { expect(JSON.parse(init.body as string)).toEqual(query); }); + test('carries the per-Type filter and sort in the POST body', async () => { + const adapter = await openAdapter(); + mockFetch.mockResolvedValueOnce(jsonResponse(queryEnvelope)); + const query = { + filter: { + contentByType: { + 'com.example/note@1': { content: { by: 'Ada' } }, + 'com.example/note@2': { content: { owner: 'Ada' } }, + }, + }, + sort: { + contentFieldByType: { + 'com.example/note@1': { field: 'by' }, + 'com.example/note@2': { value: 3 }, + }, + direction: 'asc' as const, + }, + }; + await adapter.queryRecords(query); + const [url, init] = mockFetch.mock.lastCall as [string, RequestInit]; + expect(url).toBe(`${BASE_URL}/records/query`); + expect(JSON.parse(init.body as string)).toEqual(query); + }); + test('builds correct GET params — parentId null becomes "null"', async () => { const adapter = await openAdapter(discoveryWith({ filter: { content: 'none' } })); mockFetch.mockResolvedValueOnce(jsonResponse(queryEnvelope)); @@ -1202,8 +1226,8 @@ describe('queryRecords', () => { const adapter = await openAdapter(discovery); expect(adapter.capabilities).toEqual({ - filter: { content: 'none', contentPresent: false, search: false }, - sort: { fields: [], contentField: false }, + filter: { content: 'none', contentPresent: false, contentByType: 0, search: false }, + sort: { fields: [], contentField: false, contentFieldByType: 0 }, limits: { attachmentBytes: null, contentBytes: null }, }); @@ -1263,6 +1287,20 @@ describe('queryRecords', () => { expect(mockFetch).toHaveBeenCalledTimes(1); // only the discovery call — no request sent }); + test.each([ + ['filter.contentByType', { filter: { contentByType: { 'com.example/note@1': {} } } }], + [ + 'sort.contentFieldByType', + { sort: { contentFieldByType: { 'com.example/note@1': { field: 'a' } } } }, + ], + ] as const)('refuses %s locally where the server declines it', async (capability, query) => { + const adapter = await openAdapter( + discoveryWith({ filter: { contentByType: 0 }, sort: { contentFieldByType: 0 } }), + ); + await expect(adapter.queryRecords(query)).rejects.toMatchObject({ capability }); + expect(mockFetch).toHaveBeenCalledTimes(1); // only the discovery call — no request sent + }); + test('throws APIAdapterCapabilityError for a native field outside sort.fields', async () => { const adapter = await openAdapter(discoveryWith({ sort: { fields: ['createdAt'] } })); await expect(adapter.queryRecords({ sort: { field: 'version' } })).rejects.toThrow( diff --git a/packages/adapter-api/tests/helpers.ts b/packages/adapter-api/tests/helpers.ts index e33dbb2f..b201d0fa 100644 --- a/packages/adapter-api/tests/helpers.ts +++ b/packages/adapter-api/tests/helpers.ts @@ -23,8 +23,12 @@ export const OWNER = 'entity-owner-123'; * limit and not the other is the case the null default exists for. */ export const FULL_CAPABILITIES = { - filter: { content: 'path', contentPresent: true, search: true }, - sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: true }, + filter: { content: 'path', contentPresent: true, contentByType: 64, search: true }, + sort: { + fields: ['createdAt', 'updatedAt', 'version'], + contentField: true, + contentFieldByType: 64, + }, limits: { attachmentBytes: 52428800 }, } satisfies DiscoveryCapabilities; diff --git a/packages/adapter-conformance/src/record.ts b/packages/adapter-conformance/src/record.ts index f8c55127..8ef0713c 100644 --- a/packages/adapter-conformance/src/record.ts +++ b/packages/adapter-conformance/src/record.ts @@ -76,6 +76,20 @@ export function runRecordAdapterConformance(options: RecordAdapterConformanceOpt expect(typeof capabilities.filter.search).toBe('boolean'); expect(Array.isArray(capabilities.sort.fields)).toBe(true); expect(typeof capabilities.sort.contentField).toBe('boolean'); + for (const cap of [ + capabilities.filter.contentByType, + capabilities.sort.contentFieldByType, + ]) { + expect(Number.isInteger(cap) && cap >= 0).toBe(true); + } + }); + + test('the per-Type caps are 0 whenever filter.content is "none"', () => { + // Both travel in the body of the endpoint a reach of "none" omits. + if (capabilities.filter.content === 'none') { + expect(capabilities.filter.contentByType).toBe(0); + expect(capabilities.sort.contentFieldByType).toBe(0); + } }); test('filter.contentPresent is false whenever filter.content is "none"', () => { @@ -976,6 +990,176 @@ export function runRecordAdapterConformance(options: RecordAdapterConformanceOpt }); } + // ----------------------------------------------------------------- + // Content by type — capability-gated on filter.contentByType + // ----------------------------------------------------------------- + if (capabilities.filter.contentByType >= 2) { + describe('content by type', () => { + const OTHER = 'org.haverstack.conformance/other@1'; + const THIRD = 'org.haverstack.conformance/third@1'; + + test('matches each record against the arm for its own type only', async () => { + const note = makeRecord({ content: { title: 'a' } }); + const other = makeRecord({ typeId: OTHER, content: { heading: 'a' } }); + const decoys = [ + makeRecord({ content: { title: 'b', heading: 'a' } }), + makeRecord({ typeId: OTHER, content: { title: 'a', heading: 'b' } }), + makeRecord({ typeId: THIRD, content: { title: 'a', heading: 'a' } }), + ]; + for (const record of [note, other, ...decoys]) await adapter.createRecord(record); + + const result = await adapter.queryRecords({ + filter: { + contentByType: { + [CONFORMANCE_TYPE_ID]: { content: { title: 'a' } }, + [OTHER]: { content: { heading: 'a' } }, + }, + }, + }); + expect(result.records.map((r) => r.id).sort()).toEqual([note.id, other.id].sort()); + }); + + test('an arm naming nothing matches every record of its type', async () => { + const notes = [makeRecord(), makeRecord({ content: { title: 'x' } })]; + const decoy = makeRecord({ typeId: THIRD }); + for (const record of [...notes, decoy]) await adapter.createRecord(record); + + const result = await adapter.queryRecords({ + filter: { + contentByType: { [CONFORMANCE_TYPE_ID]: {}, [OTHER]: { content: { a: 1 } } }, + }, + }); + expect(result.records.map((r) => r.id).sort()).toEqual(notes.map((r) => r.id).sort()); + }); + + test('intersects with every other key of the filter', async () => { + const kept = makeRecord({ content: { title: 'a' }, parentId: undefined }); + const parent = await adapter.createRecord(makeRecord()); + const child = makeRecord({ content: { title: 'a' }, parentId: parent.id }); + await adapter.createRecord(kept); + await adapter.createRecord(child); + + const result = await adapter.queryRecords({ + filter: { + parentId: parent.id, + contentByType: { [CONFORMANCE_TYPE_ID]: { content: { title: 'a' } }, [OTHER]: {} }, + }, + }); + expect(result.records.map((r) => r.id)).toEqual([child.id]); + }); + + if (capabilities.filter.contentPresent) { + test("an arm's contentPresent is applied to its own type", async () => { + const present = makeRecord({ content: { title: 'x', priority: 1 } }); + const absent = makeRecord({ content: { title: 'y' } }); + const other = makeRecord({ typeId: OTHER, content: { priority: 1 } }); + for (const record of [present, absent, other]) await adapter.createRecord(record); + + const result = await adapter.queryRecords({ + filter: { + contentByType: { + [CONFORMANCE_TYPE_ID]: { contentPresent: ['priority'] }, + [OTHER]: { contentPresent: ['missing'] }, + }, + }, + }); + expect(result.records.map((r) => r.id)).toEqual([present.id]); + }); + } + }); + } + + // ----------------------------------------------------------------- + // Sort by content field per type — gated on sort.contentFieldByType + // ----------------------------------------------------------------- + if (capabilities.sort.contentFieldByType >= 3) { + describe('sort by content field per type', () => { + const OTHER = 'org.haverstack.conformance/other@1'; + const THIRD = 'org.haverstack.conformance/third@1'; + const UNNAMED = 'org.haverstack.conformance/unnamed@1'; + const BY_TYPE = { + [CONFORMANCE_TYPE_ID]: { field: 'priority' }, + [OTHER]: { field: 'rank' }, + [THIRD]: { value: 2.5 }, + }; + + /** Each title names the position the record sorts to, ascending. */ + const seed = async () => { + await adapter.saveType(conformanceType()); + await adapter.saveType( + conformanceType({ + id: OTHER, + baseId: 'org.haverstack.conformance/other', + schema: { title: { kind: 'string' }, rank: { kind: 'number' } }, + }), + ); + for (const record of [ + makeRecord({ content: { title: 'd', priority: 3 } }), + makeRecord({ typeId: OTHER, content: { title: 'a', rank: 1, priority: 9 } }), + makeRecord({ typeId: THIRD, content: { title: 'c', priority: 0 } }), + makeRecord({ content: { title: 'b', priority: 2 } }), + makeRecord({ typeId: OTHER, content: { title: 'e', rank: 4 } }), + makeRecord({ content: { title: 'z' } }), + makeRecord({ typeId: UNNAMED, content: { title: 'z', priority: 1 } }), + ]) { + await adapter.createRecord(record); + } + }; + const titles = (records: { content: Record }[]) => + records.map((r) => r.content.title); + + test.each(['asc', 'desc'] as const)( + 'orders each type by its own field or constant, %s, holding no value last', + async (direction) => { + await seed(); + const result = await adapter.queryRecords({ + sort: { contentFieldByType: BY_TYPE, direction }, + }); + const ordered = ['a', 'b', 'c', 'd', 'e']; + expect(titles(result.records)).toEqual([ + ...(direction === 'asc' ? ordered : ordered.reverse()), + 'z', + 'z', + ]); + }, + ); + + test('pages through the ordering one record at a time', async () => { + await seed(); + const seen: unknown[] = []; + let cursor: string | null | undefined; + do { + const page = await adapter.queryRecords({ + limit: 1, + cursor: cursor ?? undefined, + sort: { contentFieldByType: BY_TYPE, direction: 'asc' }, + }); + seen.push(...titles(page.records)); + cursor = page.cursor; + } while (cursor); + expect(seen).toEqual(['a', 'b', 'c', 'd', 'e', 'z', 'z']); + }); + + if (capabilities.sort.contentField) { + test('a cursor minted under one field is refused under a map', async () => { + await seed(); + const minted = await adapter.queryRecords({ + limit: 1, + sort: { contentField: 'priority', direction: 'asc' }, + }); + await expectStackErrorCode( + adapter.queryRecords({ + limit: 1, + cursor: minted.cursor!, + sort: { contentFieldByType: BY_TYPE, direction: 'asc' }, + }), + 'bad_request', + ); + }); + } + }); + } + // ----------------------------------------------------------------- // Sort by content field — capability-gated on sort.contentField // ----------------------------------------------------------------- diff --git a/packages/conformance-fixtures/src/discovery.ts b/packages/conformance-fixtures/src/discovery.ts index cff6c0d3..61af2eec 100644 --- a/packages/conformance-fixtures/src/discovery.ts +++ b/packages/conformance-fixtures/src/discovery.ts @@ -21,11 +21,13 @@ export const discoveryFixtures: ConformanceFixture filter: { content: 'path', contentPresent: true, + contentByType: 16, search: true, }, sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: true, + contentFieldByType: 16, }, limits: { attachmentBytes: 52428800, @@ -47,8 +49,12 @@ export const discoveryFixtures: ConformanceFixture version: WIRE_PROTOCOL_VERSION, entityId: 'did:key:z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK', capabilities: { - filter: { content: 'none', contentPresent: false, search: false }, - sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: false }, + filter: { content: 'none', contentPresent: false, contentByType: 0, search: false }, + sort: { + fields: ['createdAt', 'updatedAt', 'version'], + contentField: false, + contentFieldByType: 0, + }, limits: { attachmentBytes: null, contentBytes: null }, }, installs: { requests: true }, @@ -70,11 +76,13 @@ export const discoveryFixtures: ConformanceFixture filter: { content: 'none', contentPresent: false, + contentByType: 0, search: false, }, sort: { fields: ['createdAt'], contentField: false, + contentFieldByType: 0, }, limits: { attachmentBytes: null, @@ -102,11 +110,13 @@ export const discoveryFixtures: ConformanceFixture filter: { content: 'path', contentPresent: true, + contentByType: 16, search: true, }, sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: true, + contentFieldByType: 16, }, limits: { attachmentBytes: 52428800, @@ -134,11 +144,13 @@ export const discoveryFixtures: ConformanceFixture filter: { content: 'path', contentPresent: true, + contentByType: 16, search: true, }, sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: true, + contentFieldByType: 16, }, limits: { attachmentBytes: 52428800, @@ -166,11 +178,13 @@ export const discoveryFixtures: ConformanceFixture filter: { content: 'none', contentPresent: false, + contentByType: 0, search: false, }, sort: { fields: ['createdAt'], contentField: false, + contentFieldByType: 0, }, limits: { attachmentBytes: null, diff --git a/packages/conformance-fixtures/src/errors.ts b/packages/conformance-fixtures/src/errors.ts index 543abeeb..24c07f6a 100644 --- a/packages/conformance-fixtures/src/errors.ts +++ b/packages/conformance-fixtures/src/errors.ts @@ -284,6 +284,44 @@ export const errorResponseFixtures: ConformanceFixture[] = [ error: { code: 'bad_request', message: 'Unknown key in filter.createdBy: authorId' }, }, }, + { + name: 'error-bad-request-empty-content-by-type', + description: + 'filter.contentByType names at least one typeId. An empty map is refused with 400 / ' + + 'code "bad_request" rather than read as no filter, so the key cannot encode to a query ' + + 'that matches everything. See docs/spec/data-model.md § Content by type.', + method: 'POST', + path: '/records/query', + requestBody: { filter: { contentByType: {} } }, + responseStatus: 400, + responseBody: { + error: { + code: 'bad_request', + message: 'filter.contentByType must name at least one typeId.', + }, + }, + }, + { + name: 'error-bad-request-content-by-type-beside-content', + description: + 'filter.contentByType never sits beside a top-level filter.content or ' + + 'filter.contentPresent: which one applies to a Record would be ambiguous, so the request ' + + 'is refused with 400 / code "bad_request" rather than resolved either way. See ' + + 'docs/spec/data-model.md § Content by type.', + method: 'POST', + path: '/records/query', + requestBody: { + filter: { content: { owner: 'Ada' }, contentByType: { 'com.example/task@1': {} } }, + }, + responseStatus: 400, + responseBody: { + error: { + code: 'bad_request', + message: + 'filter.contentByType cannot be combined with filter.content or filter.contentPresent.', + }, + }, + }, { name: 'error-bad-request-unknown-record-key', description: diff --git a/packages/conformance-fixtures/src/query.ts b/packages/conformance-fixtures/src/query.ts index e09f43d1..a16cf8ec 100644 --- a/packages/conformance-fixtures/src/query.ts +++ b/packages/conformance-fixtures/src/query.ts @@ -170,6 +170,102 @@ export const queryRecordsFixtures: ConformanceFixture< cursor: null, }, }, + { + name: 'query-filters-content-by-type', + description: + 'filter.contentByType is a content predicate keyed by typeId: a Record matches when its ' + + 'typeId is a key and its content satisfies that arm. It is how a collection carries a ' + + 'filter across the versions of a family whose migrations renamed a field — here `by` at ' + + '@1 and `owner` at @2. Types partition Records, so the arms split the query rather than ' + + 'widening it, and every other filter key applies to each arm. A server declaring ' + + 'filter.contentByType: 0 MUST refuse it with 400. See docs/spec/data-model.md § Content ' + + 'by type.', + method: 'POST', + path: '/records/query', + requestBody: { + filter: { + contentByType: { + 'com.example/task@1': { content: { by: 'Ada' } }, + 'com.example/task@2': { content: { owner: 'Ada' } }, + }, + }, + limit: 10, + }, + responseStatus: 200, + responseBody: { + records: [ + { + id: '1hk153x00061', + typeId: 'com.example/task@2', + createdAt: '2024-01-02T00:00:00.000Z', + updatedAt: '2024-01-02T00:00:00.000Z', + content: { owner: 'Ada' }, + version: 1, + }, + { + id: '1hk153x00062', + typeId: 'com.example/task@1', + createdAt: '2024-01-01T00:00:00.000Z', + updatedAt: '2024-01-01T00:00:00.000Z', + content: { by: 'Ada' }, + version: 1, + }, + ], + cursor: null, + }, + }, + { + name: 'query-sorts-by-a-content-field-per-type', + description: + 'sort.contentFieldByType names what each Type orders by: a field of its own, read as ' + + 'that Type declares it, or a constant already reduced to a number or text — here 5 for ' + + 'every @1 Record, which predates the field. A Record whose typeId is not a key, or whose ' + + 'value is null, holds no value and sorts last in either direction. A server declaring ' + + 'sort.contentFieldByType: 0 MUST refuse it with 400. See docs/spec/data-model.md ' + + '§ Sorting by a content field.', + method: 'POST', + path: '/records/query', + requestBody: { + sort: { + contentFieldByType: { + 'com.example/task@1': { value: 5 }, + 'com.example/task@2': { field: 'rank' }, + }, + direction: 'asc', + }, + limit: 3, + }, + responseStatus: 200, + responseBody: { + records: [ + { + id: '1hk153x00071', + typeId: 'com.example/task@2', + createdAt: '2024-01-02T00:00:00.000Z', + updatedAt: '2024-01-02T00:00:00.000Z', + content: { owner: 'Ada', rank: 1 }, + version: 1, + }, + { + id: '1hk153x00072', + typeId: 'com.example/task@1', + createdAt: '2024-01-01T00:00:00.000Z', + updatedAt: '2024-01-01T00:00:00.000Z', + content: { by: 'Bea' }, + version: 1, + }, + { + id: '1hk153x00073', + typeId: 'com.example/task@2', + createdAt: '2024-01-02T00:00:00.000Z', + updatedAt: '2024-01-02T00:00:00.000Z', + content: { owner: 'Cleo', rank: 9 }, + version: 1, + }, + ], + cursor: null, + }, + }, { name: 'query-content-sort-defaults-to-ascending', description: diff --git a/packages/core/README.md b/packages/core/README.md index cd7d4955..3b631ada 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -104,7 +104,7 @@ An app has two layers, because the stack draws a line between them: Migrations belong to neither. They are passed to `Stack.open()` on **every startup**, so the chain is complete before the first read. ```ts -import { Stack, migration, typeHandle } from '@haverstack/core'; +import { Stack, add, migration, typeHandle } from '@haverstack/core'; import type { Collection, ContentOf, StackClient } from '@haverstack/core'; const NoteV1 = typeHandle({ @@ -122,11 +122,9 @@ export const Note = typeHandle({ }, }); -// Startup: pass to every Stack.open(). `content` is NoteV1's, and the -// result must be Note's. -export const noteMigrations = [ - migration(NoteV1, Note, (content) => ({ ...content, pinned: false })), -]; +// Startup: pass to every Stack.open(). Steps are checked against both +// handles' schemas when migration() runs. +export const noteMigrations = [migration(NoteV1, Note, [add('pinned', false)])]; // Data layer: whoever the stack lets in. export class Notes { @@ -201,7 +199,7 @@ Tags, attachments, and relationships are unified under a single model: ### Migrations -Types can evolve over time. A new version names the one it migrates from, and `migration()` builds the step between two handles, typed on both. Pass every step to `Stack.open()`; the library composes adjacent steps into chains: +Types can evolve over time. A new version names the one it migrates from, and `migration()` builds the step between two handles from declared steps — `rename`, `map`, `add`, `drop` and `derive` — checked against both schemas when it runs. Pass every step to `Stack.open()`; the library composes adjacent steps into chains: ```ts const NoteV2 = typeHandle({ @@ -212,12 +210,12 @@ const NoteV2 = typeHandle({ }); const stack = await Stack.open(adapter, { - migrations: [migration(NoteV1, NoteV2, (content) => ({ ...content, title: '' }))], + migrations: [migration(NoteV1, NoteV2, [add('title', '')])], }); await stack.defineType(NoteV2); ``` -Records stay at the version they were written at: `get()` and `query()` return them as stored, `presentAt: 'latest'` migrates them in memory for one read, and `stack.migrateAll()` commits a family to disk. Pass migrations at every startup — see [Writing an app](#writing-an-app). +Records stay at the version they were written at: `get()` and `query()` return them as stored, `presentAt: 'latest'` migrates them in memory for one read, and `stack.migrateAll()` commits a family to disk. Because the steps are declared rather than a function, a collection carries a content filter or sort across the versions they translate to — a filter on `title` reaches records still at `@1`. Pass migrations at every startup — see [Writing an app](#writing-an-app). ## License diff --git a/packages/core/src/adapter/index.ts b/packages/core/src/adapter/index.ts index 99e4369d..d64fee51 100644 --- a/packages/core/src/adapter/index.ts +++ b/packages/core/src/adapter/index.ts @@ -31,6 +31,12 @@ export { filtersContent, parseContentFilterKey, } from '../query-validation.js'; -export { contentSortEntry, contentSortKey, compareSortEntries } from './sort.js'; +export { + contentSortEntry, + contentSortKey, + compareSortEntries, + constantSortEntry, + sortConstant, +} from './sort.js'; export type { SortEntry } from './sort.js'; export { AUTHORITY_ASSOCIATION_KINDS, partitionAssociations } from '../associations/identity.js'; diff --git a/packages/core/src/adapter/sort.ts b/packages/core/src/adapter/sort.ts index 6bff8c30..88283ccc 100644 --- a/packages/core/src/adapter/sort.ts +++ b/packages/core/src/adapter/sort.ts @@ -67,6 +67,25 @@ export function contentSortEntry(kind: ScalarFieldKind, value: unknown): SortEnt } } +/** + * A constant a `contentFieldByType` key orders by, reduced from a value of + * `kind` to what it orders as: a number, text, or null for no value. The + * reduction happens before the query is sent, so an adapter orders it + * without knowing the kind. See docs/spec/data-model.md § Sorting by a + * content field. + */ +export function sortConstant(kind: ScalarFieldKind, value: unknown): string | number | null { + const entry = contentSortEntry(kind, value); + return entry === null ? null : entry.kind === 'num' ? entry.num : entry.text; +} + +/** The entry a reduced constant orders as; see sortConstant(). */ +export function constantSortEntry(value: string | number | null): SortEntry | null { + if (typeof value === 'number') return Number.isFinite(value) ? { kind: 'num', num: value } : null; + if (typeof value === 'string') return { kind: 'text', text: value, key: contentSortKey(value) }; + return null; +} + /** * Order two strings by code point. JS `<` compares UTF-16 code units, * which files every supplementary character (U+10000 and above, stored as diff --git a/packages/core/src/collection.ts b/packages/core/src/collection.ts index 44118ba7..266c4874 100644 --- a/packages/core/src/collection.ts +++ b/packages/core/src/collection.ts @@ -27,10 +27,18 @@ import type { StackClient, } from './stack/client.js'; import type { ContentOf, ReadonlyTypeSchema, TypeHandle, UnlistedValue } from './type-handle.js'; +import { + translateArm, + translateSortKey, + UNMATCHABLE, + UNTRANSLATABLE, +} from './content-translation.js'; import type { AssociationEdit, AuthorityAssociation, ChangeFilter, + ContentArm, + ContentSortKey, DataAssociation, IfVersionOptions, NativeSortField, @@ -99,12 +107,12 @@ export type CollectionPage = { misfits: Misfit[]; cursor: string | null; /** - * A content filter or sort reaches only records at the handle's version. - * Each flag is set when records at an older or newer version match, or - * may match, the rest of the query: the results may be missing records - * the filter was meant to find. On the first page only, since it depends - * on the query rather than the page. See docs/spec/data-model.md § Type - * handles. + * A content filter or sort reaches the older versions it translates to, + * and no newer one. Each flag is set when records at a version it could + * not reach match, or may match, the rest of the query: the results may + * be missing records the filter was meant to find. On the first page + * only, since it depends on the query rather than the page. See + * docs/spec/data-model.md § Content queries across versions. */ skipped?: { older: boolean; newer: boolean }; }; @@ -160,15 +168,15 @@ export type SortableFieldOf = string extends keyof C /** * A collection reads live records of its own family only, as stored, so - * none of `typeId`, `baseId`, `includeDeleted` or `presentAt` is offered. - * Content filter keys and the sort field are checked against `C`, and - * reach only records stored at the handle's version. See - * docs/spec/data-model.md § Type handles. + * none of `typeId`, `baseId`, `includeDeleted`, `presentAt` or the per-Type + * shapes it generates itself is offered. Content filter keys and the sort + * field are checked against `C`, and carried to older versions through the + * migrations. See docs/spec/data-model.md § Type handles. */ export type CollectionQuery = Omit & { filter?: Omit< RecordFilter, - 'typeId' | 'baseId' | 'includeDeleted' | 'content' | 'contentPresent' + 'typeId' | 'baseId' | 'includeDeleted' | 'content' | 'contentPresent' | 'contentByType' > & { content?: { [E in FieldEntry as E[0]]?: E[1] | null }; contentPresent?: ContentPathOf[]; @@ -278,13 +286,16 @@ const withoutTypeId = (filter: F | undefined): Omit - Object.keys(filter?.content ?? {}).length > 0 || - (filter?.contentPresent?.length ?? 0) > 0 || - sort?.contentField !== undefined; +const contentPredicate = (filter: RecordFilter): ContentArm | undefined => { + const content = Object.keys(filter.content ?? {}).length > 0 ? filter.content : undefined; + const present = filter.contentPresent?.length ? filter.contentPresent : undefined; + if (!content && !present) return undefined; + return { ...(content && { content }), ...(present && { contentPresent: present }) }; +}; /** * The collection over `handle`, shared by `Stack.collection()` and @@ -380,31 +391,131 @@ export const createCollection = ( }; /** - * Whether a record stored at one of `typeIds` matches, or may match, - * `filter` without its content keys. A scoped page can be empty with a - * cursor set, and reading on is unbounded, so that page counts as a match. + * Whether any record matches, or may match, `filter`. A scoped page can + * be empty with a cursor set, and reading on is unbounded, so that page + * counts as a match. */ + const anyMatch = async (filter: RecordFilter): Promise => { + const page = await client.query({ filter, limit: 1 }); + return page.records.length > 0 || page.cursor !== null; + }; + + /** Whether a record stored at one of `typeIds` matches `filter` without its content keys. */ const anyAt = async (typeIds: TypeId[], filter: RecordFilter): Promise => { if (typeIds.length === 0) return false; const { content: _content, contentPresent: _present, ...rest } = filter; - const page = await client.query({ filter: { ...rest, typeId: typeIds }, limit: 1 }); - return page.records.length > 0 || page.cursor !== null; + return anyMatch({ ...rest, typeId: typeIds }); + }; + + /** + * Whether records at an older version hold what its arm left out: each + * residual is that arm with another source value in place, at its typeId. + */ + const anyResidual = async (residuals: RecordFilter[], filter: RecordFilter): Promise => { + const { content: _content, contentPresent: _present, ...rest } = filter; + const found = await Promise.all( + residuals.map((residual) => anyMatch({ ...rest, ...residual })), + ); + return found.some(Boolean); }; - const skippedVersions = async ( + /** + * The query a content filter or sort becomes: one arm per version the + * migrations carry it to, the older versions left out of it, and the + * residual predicates an arm could not ask for alongside its own. The + * first page probes the last two for `skipped.older`. Newer versions are + * always out of reach. See docs/spec/data-model.md § Content queries + * across versions. + */ + const planContentQuery = async ( filter: RecordFilter, - ): Promise['skipped']>> => { - const older: TypeId[] = []; + sort: StackQuery['sort'], + ): Promise<{ + query: Pick; + narrowed: TypeId[]; + newer: TypeId[]; + residuals: RecordFilter[]; + }> => { + const predicate = contentPredicate(filter); + const sortField = sort?.contentField; + const { capabilities } = client; + // A version past the adapter's cap is narrowed, newest kept first: + // the older a version, the fewer apps still write it. + const filterCap = + predicate === undefined + ? Infinity + : capabilities.filter.content === 'none' + ? 0 + : capabilities.filter.contentByType; + const sortCap = sortField === undefined ? Infinity : capabilities.sort.contentFieldByType; + const cap = Math.min(filterCap, sortCap); + const sortKind = (() => { + const def = sortField === undefined ? undefined : handle.schema[sortField]; + return def && def.kind !== 'array' && def.kind !== 'object' ? def.kind : undefined; + })(); + + const family = (await listTypes()) + .filter((type) => type.baseId === handle.baseId && type.version !== handleVersion) + .sort((a, b) => b.version - a.version); + const arms: Record = { [handle.id]: predicate ?? {} }; + const keys: Record = sortField + ? { [handle.id]: { field: sortField } } + : {}; + const narrowed: TypeId[] = []; const newer: TypeId[] = []; - for (const type of await listTypes()) { - if (type.baseId !== handle.baseId || type.version === handleVersion) continue; - (type.version < handleVersion ? older : newer).push(type.id); + const residuals: RecordFilter[] = []; + for (const type of family) { + if (type.version > handleVersion) { + newer.push(type.id); + continue; + } + const origins = registry.origins(type.id, handle.id); + const translated = !predicate + ? { arm: {}, residuals: [] } + : origins + ? translateArm(origins, predicate) + : UNTRANSLATABLE; + if (translated === UNMATCHABLE) continue; + const key = + sortField === undefined + ? undefined + : origins + ? translateSortKey(origins, sortField, sortKind) + : UNTRANSLATABLE; + if ( + translated === UNTRANSLATABLE || + key === UNTRANSLATABLE || + Object.keys(arms).length >= cap + ) { + narrowed.push(type.id); + continue; + } + arms[type.id] = translated.arm; + if (key) keys[type.id] = key; + for (const arm of translated.residuals) residuals.push({ ...arm, typeId: type.id }); } - const [skippedOlder, skippedNewer] = await Promise.all([ - anyAt(older, filter), - anyAt(newer, filter), - ]); - return { older: skippedOlder, newer: skippedNewer }; + + const { content: _content, contentPresent: _present, ...rest } = filter; + const typeIds = Object.keys(arms); + if (typeIds.length === 1) { + return { + query: { filter: { ...filter, typeId: handle.id }, sort }, + narrowed, + newer, + residuals, + }; + } + return { + query: { + filter: predicate ? { ...rest, contentByType: arms } : { ...rest, typeId: typeIds }, + sort: sortField + ? { contentFieldByType: keys, ...(sort?.direction && { direction: sort.direction }) } + : sort, + }, + narrowed, + newer, + residuals, + }; }; const mutate = async ( @@ -440,22 +551,33 @@ export const createCollection = ( // Stripped as well as untyped: either would change which records a // page holds, or the shape they arrive in. const { presentAt: _presentAt, filter, ...rest } = query as StackQuery; - const narrowed = readsContent(filter, rest.sort); // A concrete typeId needs no family lookup; baseId does. - const { baseId: _baseId, ...unversioned } = withoutTypeId(filter); - const [{ records, cursor }, skipped] = await Promise.all([ - client.query({ + const { + baseId: _baseId, + contentByType: _byType, + ...unversioned + } = withoutTypeId(filter) as RecordFilter; + const firstPage = rest.cursor === undefined; + let records: StackRecord[]; + let cursor: string | null; + let skipped: CollectionPage['skipped']; + if (!contentPredicate(unversioned) && rest.sort?.contentField === undefined) { + ({ records, cursor } = await client.query({ ...rest, - filter: narrowed - ? { ...unversioned, typeId: handle.id } - : { ...unversioned, baseId: handle.baseId }, - }), - rest.cursor !== undefined - ? undefined - : narrowed - ? skippedVersions(unversioned) - : { older: false, newer: false }, - ]); + filter: { ...unversioned, baseId: handle.baseId }, + })); + if (firstPage) skipped = { older: false, newer: false }; + } else { + const plan = await planContentQuery(unversioned, rest.sort); + const [page, narrowed, residual, newer] = await Promise.all([ + client.query({ ...rest, ...plan.query }), + firstPage && anyAt(plan.narrowed, unversioned), + firstPage && anyResidual(plan.residuals, unversioned), + firstPage && anyAt(plan.newer, unversioned), + ]); + ({ records, cursor } = page); + if (firstPage) skipped = { older: narrowed || residual, newer }; + } const page: CollectionPage = { records: [], misfits: [], diff --git a/packages/core/src/content-match.ts b/packages/core/src/content-match.ts new file mode 100644 index 00000000..5b2857b6 --- /dev/null +++ b/packages/core/src/content-match.ts @@ -0,0 +1,61 @@ +/** + * Content predicates, evaluated in memory + * ------------------------------------------------------- + * How a `content` filter value and a `contentPresent` path are matched + * against one content object: element-wise through arrays, with `null` + * standing for "no value". MemoryAdapter answers queries with it, and a + * collection decides a constant a migration added the same way, so a key + * dropped in core matches exactly what an adapter would have matched. See + * docs/spec/data-model.md § Nested content paths. + */ + +import { parseContentFilterKey } from './query-validation.js'; +import type { ContentArm } from './types/index.js'; + +/** An array stands for its elements; anything else stands for itself. */ +const spreadValue = (value: unknown): unknown[] => (Array.isArray(value) ? value : [value]); + +/** + * Every value reachable by walking `segments`, spreading arrays at each + * step — sqlite-shared's contentPathExists() walk, so a filter answers the + * same against either. Empty means the path reaches nothing, which a `null` + * filter value matches. + */ +export const valuesAtContentPath = ( + content: Record, + segments: string[], +): unknown[] => { + let current: unknown[] = [content]; + for (const segment of segments) { + const next: unknown[] = []; + for (const value of current) { + for (const item of spreadValue(value)) { + if (typeof item !== 'object' || item === null || Array.isArray(item)) continue; + if (Object.hasOwn(item, segment)) next.push((item as Record)[segment]); + } + } + current = next; + } + // The leaf spreads too, so a filter on `tags` matches a record whose + // `tags` array contains the value. + return current.flatMap(spreadValue); +}; + +/** + * Whether `content` satisfies a `content` and `contentPresent` predicate. A + * `null` filter value means "field absent or null", not "match nothing": + * plain `===` would miss an absent field (docs/spec/data-model.md § Filter). + */ +export const matchesContent = (content: Record, predicate: ContentArm): boolean => + Object.entries(predicate.content ?? {}).every(([key, value]) => { + const found = valuesAtContentPath(content, parseContentFilterKey(key)); + if (value === null || value === undefined) { + return found.length === 0 || found.some((v) => v === null || v === undefined); + } + return found.some((v) => v === value); + }) && + (predicate.contentPresent ?? []).every((key) => + valuesAtContentPath(content, parseContentFilterKey(key)).some( + (v) => v !== null && v !== undefined, + ), + ); diff --git a/packages/core/src/content-translation.ts b/packages/core/src/content-translation.ts new file mode 100644 index 00000000..6624c926 --- /dev/null +++ b/packages/core/src/content-translation.ts @@ -0,0 +1,132 @@ +/** + * Content queries across versions + * ------------------------------------------------------- + * A collection's content filter and sort are written in its handle's + * shape, and an adapter matches content as stored. These functions rewrite + * one predicate or sort field into an older version's shape, through the + * field origins a migration chain declares, or say why they cannot: an arm + * no record at that version can match is dropped, and one that does not + * translate is narrowed. See docs/spec/data-model.md § Content queries + * across versions. + */ + +import { matchesContent } from './content-match.js'; +import { sortConstant } from './adapter/sort.js'; +import { parseContentFilterKey } from './query-validation.js'; +import type { FieldOrigin, FieldOrigins } from './migration-steps.js'; +import type { ContentArm, ContentSortKey, ScalarFieldKind } from './types/index.js'; + +/** No record at the version can match, so its arm is dropped without a skip. */ +export const UNMATCHABLE = 'unmatchable'; +/** The version is narrowed: left out of the query, and reported if it matches the rest. */ +export const UNTRANSLATABLE = 'untranslatable'; + +type Untranslated = typeof UNMATCHABLE | typeof UNTRANSLATABLE; + +/** + * The older values a `map` sends to `value`, mapped ones first. A value no + * entry names maps to itself, and `map` reaches strings only. + */ +const sourcesOf = (values: Readonly>, value: unknown): unknown[] => { + if (typeof value !== 'string') return [value]; + const sources: unknown[] = Object.keys(values).filter((from) => values[from] === value); + if (!Object.hasOwn(values, value)) sources.push(value); + return sources; +}; + +/** + * Whether a constant satisfies one key, decided as an adapter would decide + * it against a record holding the constant at `field`. + */ +const constantMatches = ( + field: string, + origin: Extract, + predicate: ContentArm, +): boolean => + matchesContent(origin.value === undefined ? {} : { [field]: origin.value }, predicate); + +/** + * An older version's arm, and the arms it leaves out: when two older + * values map to the one a key asks for, the arm asks for the first and + * each other one becomes a residual, which the first page probes for + * `skipped.older` rather than narrowing the version outright. + */ +export type TranslatedArm = { arm: ContentArm; residuals: ContentArm[] }; + +/** + * `predicate`, written in the handle's shape, as the arm for the version + * `origins` describes. A key is translated by its top-level field: the + * rest of its path is kept, since steps reach top-level fields only. + */ +export const translateArm = ( + origins: FieldOrigins, + predicate: ContentArm, +): TranslatedArm | Untranslated => { + const content: Record = {}; + const present: string[] = []; + let split: { path: string; others: unknown[] } | undefined; + const keys: [string, unknown, boolean][] = [ + ...Object.entries(predicate.content ?? {}).map( + ([key, value]) => [key, value, false] as [string, unknown, boolean], + ), + ...(predicate.contentPresent ?? []).map( + (key) => [key, undefined, true] as [string, unknown, boolean], + ), + ]; + for (const [key, value, presence] of keys) { + const [field, ...rest] = parseContentFilterKey(key); + const origin = origins.get(field); + if (!origin || origin.kind === 'derived') return UNTRANSLATABLE; + if (origin.kind === 'constant') { + const asked = presence ? { contentPresent: [key] } : { content: { [key]: value } }; + if (!constantMatches(field, origin, asked)) return UNMATCHABLE; + continue; + } + const path = [origin.field, ...rest].join('.'); + if (presence) { + present.push(path); + continue; + } + // A path below a mapped field reaches no string, so only the field's + // own values are remapped; `null` is never one of them. + if (!origin.values || rest.length > 0 || value === null || value === undefined) { + content[path] = value; + continue; + } + const [first, ...others] = sourcesOf(origin.values, value); + if (first === undefined) return UNMATCHABLE; + // One split per arm keeps the probes to one per source; two would + // multiply them. + if (others.length > 0) { + if (split) return UNTRANSLATABLE; + split = { path, others }; + } + content[path] = first; + } + const arm = { + ...(Object.keys(content).length > 0 && { content }), + ...(present.length > 0 && { contentPresent: present }), + }; + const residuals = (split?.others ?? []).map((other) => ({ + ...arm, + content: { ...content, [split!.path]: other }, + })); + return { arm, residuals }; +}; + +/** + * What a record at the version `origins` describes orders by, for a sort + * on the handle's `field` of `kind`. A remapped field is narrowed, since + * remapped values don't keep their order. + */ +export const translateSortKey = ( + origins: FieldOrigins, + field: string, + kind: ScalarFieldKind | undefined, +): ContentSortKey | typeof UNTRANSLATABLE => { + const origin = origins.get(field); + if (!origin || !kind || origin.kind === 'derived') return UNTRANSLATABLE; + if (origin.kind === 'constant') return { value: sortConstant(kind, origin.value) }; + if (origin.values && Object.keys(origin.values).length > 0) return UNTRANSLATABLE; + return { field: origin.field }; +}; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index df53481f..eb55fc16 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -40,6 +40,15 @@ export { isPlanEmpty } from './install.js'; // Type handles export { typeHandle, migration } from './type-handle.js'; +export { rename, map, add, drop, derive } from './migration-steps.js'; +export type { + MigrationStep, + RenameStep, + MapStep, + AddStep, + DropStep, + DeriveStep, +} from './migration-steps.js'; export type { TypeHandle, Migration, @@ -140,6 +149,8 @@ export type { AttachmentFilter, RelationshipTargetPattern, QuerySort, + ContentArm, + ContentSortKey, NativeSortField, QueryResult, DateRange, diff --git a/packages/core/src/migration-steps.ts b/packages/core/src/migration-steps.ts new file mode 100644 index 00000000..80e9facf --- /dev/null +++ b/packages/core/src/migration-steps.ts @@ -0,0 +1,365 @@ +/** + * Migration steps + * ------------------------------------------------------- + * A migration is a list of declared steps — rename, map, add, drop and + * derive — rather than a function, so that core can read what became of + * each field. Two things are built from one list: the in-memory `migrate` + * that `Stack.open()`, a collection's reads and `migrateAll()` run, and the + * field origins a collection translates a content filter or sort through. + * + * `fieldOrigins()` also checks the list against both schemas, so a step + * the target Type would refuse fails when `migration()` runs rather than + * on the first record `migrateAll()` writes. See docs/spec/data-model.md + * § Migration steps. + */ + +import type { FieldDef, TypeSchema } from './types/index.js'; +import type { ReadonlyTypeSchema } from './type-handle.js'; +import { validateContent } from './validate.js'; + +// ------------------------------------------------------- +// Steps +// ------------------------------------------------------- + +export type RenameStep = { readonly op: 'rename'; readonly from: string; readonly to: string }; +export type MapStep = { + readonly op: 'map'; + readonly field: string; + readonly values: Readonly>; +}; +export type AddStep = { readonly op: 'add'; readonly field: string; readonly value: unknown }; +export type DropStep = { readonly op: 'drop'; readonly field: string }; +/** + * `derive` is a method so a step typed on one handle's content fits a list + * of untyped ones, as `Migration.migrate` does. + */ +export type DeriveStep> = { + readonly op: 'derive'; + readonly field: string; + derive(content: F): unknown; +}; + +/** One step of a migration, typed on the content stored at its `from`. */ +export type MigrationStep> = + | RenameStep + | MapStep + | AddStep + | DropStep + | DeriveStep; + +/** Move a field's value to a new name. */ +export const rename = (from: string, to: string): RenameStep => + Object.freeze({ op: 'rename', from, to }); + +/** + * Replace each value `values` lists with its mapping, keeping any other, + * element-wise through an array. A stored enum may hold a value no handle + * lists, and it is carried through rather than lost. + */ +export const map = (field: string, values: Readonly>): MapStep => + Object.freeze({ op: 'map', field, values: Object.freeze({ ...values }) }); + +/** Set a field to a constant. */ +export const add = (field: string, value: unknown): AddStep => + Object.freeze({ op: 'add', field, value }); + +/** Remove a field. */ +export const drop = (field: string): DropStep => Object.freeze({ op: 'drop', field }); + +/** + * Set a field to what `fn` computes from the record as stored at the + * migration's `from`; `undefined` leaves it absent. The one opaque step: a + * field it produces does not translate across versions. + */ +export const derive = (field: string, fn: (content: F) => unknown): DeriveStep => + Object.freeze({ op: 'derive', field, derive: fn }); + +const describe = (step: MigrationStep): string => { + switch (step.op) { + case 'rename': + return `rename("${step.from}", "${step.to}")`; + case 'add': + return `add("${step.field}", …)`; + default: + return `${step.op}("${step.field}"${step.op === 'drop' ? '' : ', …'})`; + } +}; + +// ------------------------------------------------------- +// Applying steps +// ------------------------------------------------------- + +const remap = (value: unknown, values: Readonly>): unknown => { + if (Array.isArray(value)) return value.map((v) => remap(v, values)); + return typeof value === 'string' && Object.hasOwn(values, value) ? values[value] : value; +}; + +/** The content `steps` produce from `content`, which is left untouched. */ +export const applySteps = ( + steps: readonly MigrationStep[], + content: Record, +): Record => { + const out: Record = { ...content }; + for (const step of steps) { + switch (step.op) { + case 'rename': + if (Object.hasOwn(out, step.from)) out[step.to] = out[step.from]; + delete out[step.from]; + break; + case 'map': + if (out[step.field] !== undefined) out[step.field] = remap(out[step.field], step.values); + break; + case 'add': + out[step.field] = step.value; + break; + case 'drop': + delete out[step.field]; + break; + case 'derive': { + const value = step.derive(content); + if (value === undefined) delete out[step.field]; + else out[step.field] = value; + break; + } + } + } + return out; +}; + +// ------------------------------------------------------- +// Field origins +// ------------------------------------------------------- + +/** + * What one of the target's top-level fields is, in terms of the source's: + * a source field, its string values possibly remapped; a constant, where + * `undefined` is no value; or derived, which nothing can see through. + */ +export type FieldOrigin = + | { + readonly kind: 'field'; + readonly field: string; + readonly values?: Readonly>; + } + | { readonly kind: 'constant'; readonly value: unknown } + | { readonly kind: 'derived' }; + +/** Every top-level field the target declares, by its origin. */ +export type FieldOrigins = ReadonlyMap; + +/** `outer` after `inner`, as one table over both domains. */ +const composeValues = ( + inner: Readonly> | undefined, + outer: Readonly> | undefined, +): Readonly> | undefined => { + if (!inner) return outer; + if (!outer) return inner; + const composed: Record = {}; + for (const [from, to] of Object.entries(inner)) composed[from] = remap(to, outer) as string; + for (const [from, to] of Object.entries(outer)) { + if (!Object.hasOwn(inner, from)) composed[from] = to; + } + return composed; +}; + +/** + * The origins across two migrations end to end: `lower` takes A to B and + * `upper` takes B to C, so the result says what each of C's fields is in + * terms of A's. + */ +export const composeOrigins = (lower: FieldOrigins, upper: FieldOrigins): FieldOrigins => { + const composed = new Map(); + for (const [field, origin] of upper) { + if (origin.kind !== 'field') { + composed.set(field, origin); + continue; + } + const inner = lower.get(origin.field) ?? { kind: 'derived' }; + composed.set( + field, + inner.kind === 'field' + ? { kind: 'field', field: inner.field, values: composeValues(inner.values, origin.values) } + : inner.kind === 'constant' + ? { + kind: 'constant', + value: origin.values ? remap(inner.value, origin.values) : inner.value, + } + : inner, + ); + } + return composed; +}; + +const isStringKind = (def: FieldDef | undefined): boolean => + def?.kind === 'enum' || def?.kind === 'string' || def?.kind === 'text'; + +/** What `map` may remap: a string-shaped scalar, or a declared array of one. */ +const isMappable = (def: FieldDef | undefined): boolean => + isStringKind(def) || (def?.kind === 'array' && !def.open && isStringKind(def.items)); + +const MAX_CARRY_DEPTH = 32; + +/** + * Why a value stored under `source` could fail `target` once `values` + * remaps it, or null when it cannot. A carried field is written as it + * stands, so this is write validity, not the read compatibility + * isCompatible() answers. + */ +const carryProblem = ( + path: string, + source: FieldDef, + target: FieldDef, + values: Readonly> | undefined, + depth: number, +): string | null => { + if (depth > MAX_CARRY_DEPTH) return `"${path}" nests too deeply to check`; + if (target.required && !source.required) { + return `"${path}" is optional at the source and required at the target`; + } + if (target.kind === 'array') { + if (source.kind !== 'array') return `"${path}" changes kind from ${source.kind} to array`; + if (target.open) return null; + if (source.open) return `"${path}" is an open array at the source and declared at the target`; + return carryProblem(`${path}[]`, source.items, target.items, values, depth + 1); + } + if (target.kind === 'object') { + if (source.kind !== 'object') return `"${path}" changes kind from ${source.kind} to object`; + if (target.open) return null; + if (source.open) return `"${path}" is an open object at the source and declared at the target`; + for (const [key, def] of Object.entries(target.properties)) { + const from = source.properties[key]; + if (!from) { + if (def.required) return `"${path}.${key}" is required at the target and not at the source`; + continue; + } + const problem = carryProblem(`${path}.${key}`, from, def, undefined, depth + 1); + if (problem) return problem; + } + for (const key of Object.keys(source.properties)) { + if (!Object.hasOwn(target.properties, key)) { + return `"${path}.${key}" is not declared at the target`; + } + } + return null; + } + if (target.kind === 'enum') { + if (source.kind !== 'enum') return `"${path}" changes kind from ${source.kind} to enum`; + const allowed = new Set(target.values); + const missing = source.values.find((v) => !allowed.has(remap(v, values ?? {}) as string)); + return missing === undefined + ? null + : `"${path}" holds "${missing}"${values && Object.hasOwn(values, missing) ? ` (mapped to "${values[missing]}")` : ''}, which the target's enum does not list`; + } + if (target.kind === 'string' || target.kind === 'text') { + return isStringKind(source) + ? null + : `"${path}" changes kind from ${source.kind} to ${target.kind}`; + } + return source.kind === target.kind + ? null + : `"${path}" changes kind from ${source.kind} to ${target.kind}`; +}; + +/** + * What each of `to`'s top-level fields is in terms of `from`'s, after + * `steps`. Returns the problem that makes the list unusable as a string + * instead: a step naming a field that is not there, a field `to` would + * refuse, or one it requires that nothing produces. See + * docs/spec/data-model.md § Migration steps. + */ +export const fieldOrigins = ( + steps: readonly MigrationStep[], + from: ReadonlyTypeSchema, + to: ReadonlyTypeSchema, +): FieldOrigins | string => { + const current = new Map( + Object.keys(from).map((field) => [field, { kind: 'field', field }]), + ); + for (const step of steps) { + const at = describe(step); + const named = step.op === 'rename' ? step.from : step.field; + if (step.op !== 'add' && step.op !== 'derive' && !current.has(named)) { + return `${at}: "${named}" is not a field at this point`; + } + switch (step.op) { + case 'rename': + if (current.has(step.to)) return `${at}: "${step.to}" is already a field at this point`; + current.set(step.to, current.get(step.from)!); + current.delete(step.from); + break; + case 'map': { + const origin = current.get(step.field)!; + if (origin.kind === 'field') { + if (!isMappable(from[origin.field] as FieldDef | undefined)) { + return `${at}: only an enum, string or text field, or an array of one, can be mapped`; + } + current.set(step.field, { + ...origin, + values: composeValues(origin.values, step.values), + }); + } else if (origin.kind === 'constant') { + current.set(step.field, { kind: 'constant', value: remap(origin.value, step.values) }); + } + break; + } + case 'add': + if (current.has(step.field)) + return `${at}: "${step.field}" is already a field at this point`; + if (step.value === undefined) return `${at}: an added field needs a value`; + current.set(step.field, { kind: 'constant', value: step.value }); + break; + case 'drop': + current.delete(step.field); + break; + case 'derive': + current.set(step.field, { kind: 'derived' }); + break; + } + } + + for (const field of current.keys()) { + if (!Object.hasOwn(to, field)) { + return `"${field}" is carried to a Type that does not declare it; drop or rename it`; + } + } + for (const [field, def] of Object.entries(to)) { + const origin = current.get(field); + if (!origin) { + if (def.required) return `"${field}" is required at the target, and no step produces it`; + current.set(field, { kind: 'constant', value: undefined }); + continue; + } + if (origin.kind === 'field') { + const problem = carryProblem( + field, + from[origin.field] as FieldDef, + def as FieldDef, + origin.values, + 0, + ); + if (problem) return problem; + } else if (origin.kind === 'constant') { + const errors = validateContent({ [field]: origin.value }, { [field]: def } as TypeSchema); + if (errors.length > 0) + return `"${field}" is set to a value the target refuses: ${errors[0].message}`; + } + } + return current; +}; + +// ------------------------------------------------------- +// Registry of origins +// ------------------------------------------------------- + +/** + * The origins `migration()` computed for each migration it built. A + * migration assembled by hand has none, and a collection reads every field + * across it as derived. + */ +const ORIGINS = new WeakMap(); + +export const recordOrigins = (migration: object, origins: FieldOrigins): void => { + ORIGINS.set(migration, origins); +}; + +export const originsOf = (migration: object): FieldOrigins | undefined => ORIGINS.get(migration); diff --git a/packages/core/src/query-validation.ts b/packages/core/src/query-validation.ts index 47efa11a..8a526ced 100644 --- a/packages/core/src/query-validation.ts +++ b/packages/core/src/query-validation.ts @@ -17,7 +17,7 @@ */ import { ARGUMENTS_INVALID, StackBadRequestError, StackValidationError } from './errors.js'; -import { familyIdProblem } from './schema.js'; +import { familyIdProblem, parseTypeId } from './schema.js'; import { targetErrors } from './associations/validation.js'; import { CONTENT_SEGMENT_METACHARACTERS, SEGMENT_METACHARACTER_RE } from './validate.js'; import { NATIVE_SORT_FIELDS } from './types/index.js'; @@ -40,32 +40,51 @@ export function assertQueryCapabilities( filter: RecordFilter | undefined, capabilities: Pick, ): void { - const { content: reach, contentPresent, search } = capabilities.filter; + const { content: reach, contentPresent, contentByType, search } = capabilities.filter; if (filter?.search && !search) { throw new StackBadRequestError( 'Query uses filter.search, but this adapter does not declare the filter.search capability.', 'filter.search', ); } - const present = filter?.contentPresent?.length ? filter.contentPresent : undefined; - if (!filter?.content && !present) return; + // An arm's keys are gated as the same keys at the top of the filter are. + const arms = Object.values(filter?.contentByType ?? {}); + const parts = [filter ?? {}, ...arms]; + const keys = parts.flatMap((part) => Object.keys(part.content ?? {})); + const present = parts.flatMap((part) => part.contentPresent ?? []); + if (!filter?.content && present.length === 0 && arms.length === 0) return; if (reach === 'none') { // Named as `filter.content` whichever key the query used: the reach // is what is absent, and it is the entry to look at in discovery. + const used = + arms.length > 0 + ? 'filter.contentByType' + : present.length > 0 && !filter?.content + ? 'filter.contentPresent' + : 'filter.content'; throw new StackBadRequestError( - `Query uses ${present && !filter?.content ? 'filter.contentPresent' : 'filter.content'}, ` + - 'but this adapter declares filter.content: "none".', + `Query uses ${used}, but this adapter declares filter.content: "none".`, 'filter.content', ); } - if (present && !contentPresent) { + if (arms.length > contentByType) { + throw new StackBadRequestError( + contentByType === 0 + ? 'Query uses filter.contentByType, but this adapter does not declare the ' + + 'filter.contentByType capability.' + : `Query uses ${arms.length} filter.contentByType arms, but this adapter takes at ` + + `most ${contentByType}.`, + 'filter.contentByType', + ); + } + if (present.length > 0 && !contentPresent) { throw new StackBadRequestError( 'Query uses filter.contentPresent, but this adapter does not declare the ' + 'filter.contentPresent capability.', 'filter.contentPresent', ); } - for (const key of [...Object.keys(filter?.content ?? {}), ...(present ?? [])]) { + for (const key of [...keys, ...present]) { if (parseContentFilterKey(key).length > 1 && reach !== 'path') { throw new StackBadRequestError( `Query uses the nested content path "${key}", but this adapter declares ` + @@ -76,6 +95,66 @@ export function assertQueryCapabilities( } } +const isPlainObject = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +/** + * Hold `filter.contentByType` to a split along Types: at least one arm, + * each pinned to a TypeId and flat, and never beside a top-level content + * predicate, whose relation to the arms would be ambiguous. A map that + * could arrive empty would match everything once it crossed the wire. See + * docs/spec/data-model.md § Content by type. + */ +export function assertValidContentByType(filter: RecordFilter | undefined): void { + const arms: unknown = filter?.contentByType; + if (arms === undefined) return; + if (!isPlainObject(arms) || Object.keys(arms).length === 0) { + throw new StackBadRequestError('filter.contentByType must name at least one typeId.'); + } + if (filter!.content !== undefined || filter!.contentPresent !== undefined) { + throw new StackBadRequestError( + 'filter.contentByType cannot be combined with filter.content or filter.contentPresent.', + ); + } + for (const [typeId, arm] of Object.entries(arms)) { + if (!parseTypeId(typeId)) { + throw new StackBadRequestError( + `filter.contentByType key "${typeId}" is not a versioned TypeId.`, + ); + } + if (!isPlainObject(arm)) { + throw new StackBadRequestError(`filter.contentByType["${typeId}"] must be an object.`); + } + for (const key of Object.keys(arm)) { + if (key !== 'content' && key !== 'contentPresent') { + throw new StackBadRequestError( + `filter.contentByType["${typeId}"] takes content and contentPresent only, not "${key}".`, + ); + } + } + if (arm.content !== undefined && !isPlainObject(arm.content)) { + throw new StackBadRequestError( + `filter.contentByType["${typeId}"].content must be an object.`, + ); + } + if ( + arm.contentPresent !== undefined && + (!Array.isArray(arm.contentPresent) || + !arm.contentPresent.every((path) => typeof path === 'string')) + ) { + throw new StackBadRequestError( + `filter.contentByType["${typeId}"].contentPresent must be an array of strings.`, + ); + } + for (const key of [ + ...Object.keys((arm.content as Record | undefined) ?? {}), + ...((arm.contentPresent as string[] | undefined) ?? []), + ]) { + parseContentFilterKey(key); + } + } +} + /** * Whether a single-segment content filter can be pushed down to the * adapter. Callers that read a content field pair this with an in-memory @@ -141,9 +220,12 @@ const VALID_SORT_DIRECTIONS = new Set(['asc', 'desc']); */ export function assertValidSort(sort: QuerySort | undefined): void { if (!sort) return; - if (sort.field !== undefined && sort.contentField !== undefined) { + const named = [sort.field, sort.contentField, sort.contentFieldByType].filter( + (member) => member !== undefined, + ); + if (named.length > 1) { throw new StackBadRequestError( - 'A sort names either a native field or a content field, never both.', + 'A sort names one of a native field, a content field or a content field per type.', ); } if (sort.field !== undefined && !VALID_SORT_FIELDS.has(sort.field)) { @@ -151,18 +233,8 @@ export function assertValidSort(sort: QuerySort | undefined): void { `Invalid sort field "${sort.field}": expected one of ${NATIVE_SORT_FIELDS.join(', ')}.`, ); } - if (sort.contentField !== undefined) { - // Parsed by the same rule a filter key is, so a name a filter could - // never address is not one a sort can either — then held to one - // segment, because a value inside an array or object has no single - // position to order its record by (docs/spec/data-model.md - // § Sorting by a content field). - if (parseContentFilterKey(sort.contentField).length > 1) { - throw new StackBadRequestError( - `Invalid sort content field "${sort.contentField}": sorting reaches top-level fields only.`, - ); - } - } + if (sort.contentField !== undefined) assertSortableField(sort.contentField); + if (sort.contentFieldByType !== undefined) assertValidSortByType(sort.contentFieldByType); if (sort.direction !== undefined && !VALID_SORT_DIRECTIONS.has(sort.direction)) { throw new StackBadRequestError( `Invalid sort direction "${sort.direction}": expected "asc" or "desc".`, @@ -170,6 +242,58 @@ export function assertValidSort(sort: QuerySort | undefined): void { } } +/** + * Parsed by the same rule a filter key is, so a name a filter could never + * address is not one a sort can either — then held to one segment, because + * a value inside an array or object has no single position to order its + * record by (docs/spec/data-model.md § Sorting by a content field). + */ +const assertSortableField = (field: unknown): void => { + if (typeof field !== 'string') { + throw new StackBadRequestError('Invalid sort content field: expected a string.'); + } + if (parseContentFilterKey(field).length > 1) { + throw new StackBadRequestError( + `Invalid sort content field "${field}": sorting reaches top-level fields only.`, + ); + } +}; + +/** + * Each key names a TypeId and orders by one field of it or one reduced + * constant. See docs/spec/data-model.md § Sorting by a content field. + */ +const assertValidSortByType = (byType: unknown): void => { + if (!isPlainObject(byType) || Object.keys(byType).length === 0) { + throw new StackBadRequestError('sort.contentFieldByType must name at least one typeId.'); + } + for (const [typeId, key] of Object.entries(byType)) { + if (!parseTypeId(typeId)) { + throw new StackBadRequestError( + `sort.contentFieldByType key "${typeId}" is not a versioned TypeId.`, + ); + } + const shape = isPlainObject(key) ? Object.keys(key) : []; + if (shape.length !== 1 || (shape[0] !== 'field' && shape[0] !== 'value')) { + throw new StackBadRequestError( + `sort.contentFieldByType["${typeId}"] must be { field } or { value }.`, + ); + } + const { field, value } = key as { field?: unknown; value?: unknown }; + if (shape[0] === 'field') { + assertSortableField(field); + } else if ( + value !== null && + typeof value !== 'string' && + !(typeof value === 'number' && Number.isFinite(value)) + ) { + throw new StackBadRequestError( + `sort.contentFieldByType["${typeId}"].value must be a string, a finite number or null.`, + ); + } + } +}; + /** * The sort every adapter receives: a named sort with no `direction` is * ascending, resolved here so no adapter carries its own default. No sort @@ -198,6 +322,20 @@ export function assertSortCapability( capabilities: Pick, ): void { if (!sort) return; + if (sort.contentFieldByType !== undefined) { + const keys = Object.keys(sort.contentFieldByType).length; + const cap = capabilities.sort.contentFieldByType; + if (keys > cap) { + throw new StackBadRequestError( + cap === 0 + ? 'Query uses sort.contentFieldByType, but this adapter does not declare the ' + + 'sort.contentFieldByType capability.' + : `Query uses ${keys} sort.contentFieldByType keys, but this adapter takes at most ${cap}.`, + 'sort.contentFieldByType', + ); + } + return; + } if (sort.contentField !== undefined) { if (!capabilities.sort.contentField) { throw new StackBadRequestError( diff --git a/packages/core/src/stack/migrations.ts b/packages/core/src/stack/migrations.ts index c0327350..9a71bd12 100644 --- a/packages/core/src/stack/migrations.ts +++ b/packages/core/src/stack/migrations.ts @@ -13,6 +13,8 @@ import { parseTypeId } from '../schema.js'; import { StackMigrationError } from '../errors.js'; import type { StackRecord, TypeId } from '../types/index.js'; import type { Migration } from '../type-handle.js'; +import { composeOrigins, originsOf } from '../migration-steps.js'; +import type { FieldOrigins } from '../migration-steps.js'; export class MigrationRegistry { private readonly steps = new Map(); @@ -91,6 +93,24 @@ export class MigrationRegistry { return (content) => steps.reduce((c, step) => step.migrate(c), content); } + /** + * What each of `toId`'s top-level fields is in terms of `fromId`'s, + * composed along the path between them. Null when there is no path, or + * when a step on it was not built by `migration()` and so declares + * nothing. See docs/spec/data-model.md § Content queries across versions. + */ + origins(fromId: TypeId, toId: TypeId): FieldOrigins | null { + let composed: FieldOrigins | null = null; + for (let current = fromId; current !== toId; ) { + const step = this.steps.get(current); + const origins = step && originsOf(step); + if (!origins) return null; + composed = composed ? composeOrigins(composed, origins) : origins; + current = step.to; + } + return composed; + } + /** * Find the latest registered version of a type family. * Follows the migration chain from the given typeId to the end. diff --git a/packages/core/src/stack/stack.ts b/packages/core/src/stack/stack.ts index e2cf121a..c89afa6e 100644 --- a/packages/core/src/stack/stack.ts +++ b/packages/core/src/stack/stack.ts @@ -97,6 +97,7 @@ import { } from '../errors.js'; import { assertQueryCapabilities, + assertValidContentByType, assertSortCapability, assertValidAssociationFilters, assertValidBaseIdFilter, @@ -1195,6 +1196,7 @@ export class Stack implements StackClient { async query(query: StackQuery = {}): Promise { this.assertOpen(); const { presentAt, filter, limit: rawLimit, ...rest } = query; + assertValidContentByType(filter); assertQueryCapabilities(filter, this.adapter.capabilities); assertValidSort(query.sort); assertSortCapability(query.sort, this.adapter.capabilities); diff --git a/packages/core/src/testing.ts b/packages/core/src/testing.ts index 4028f9f9..31daa1b9 100644 --- a/packages/core/src/testing.ts +++ b/packages/core/src/testing.ts @@ -38,7 +38,7 @@ import type { import { SYSTEM_TYPES } from './types/index.js'; import { applyMergePatch } from './merge.js'; import { fileRefFields } from './schema.js'; -import { compareSortEntries, contentSortEntry } from './adapter/sort.js'; +import { compareSortEntries, constantSortEntry, contentSortEntry } from './adapter/sort.js'; import type { SortEntry } from './adapter/sort.js'; import { @@ -47,7 +47,7 @@ import { StackNotFoundError, StackBadRequestError, } from './errors.js'; -import { parseContentFilterKey } from './query-validation.js'; +import { matchesContent } from './content-match.js'; import { applyAssociationEdits, assertOneSurface, @@ -62,35 +62,9 @@ const requireDirection = (sort: QuerySort): 'asc' | 'desc' => { } return sort.direction; }; -/** An array stands for its elements; anything else stands for itself. */ -const spreadValue = (value: unknown): unknown[] => (Array.isArray(value) ? value : [value]); - /** A FileId is the SHA-256 hex digest of its bytes — see docs/spec/attachments.md. */ const SHA256_HEX_RE = /^[0-9a-f]{64}$/; -/** - * Every value reachable by walking `segments`, spreading arrays at each - * step — sqlite-shared's contentPathExists() walk, so a filter answers the - * same against either. Empty means the path reaches nothing, which a `null` - * filter value matches. See docs/spec/data-model.md § Nested content paths. - */ -const valuesAtContentPath = (content: Record, segments: string[]): unknown[] => { - let current: unknown[] = [content]; - for (const segment of segments) { - const next: unknown[] = []; - for (const value of current) { - for (const item of spreadValue(value)) { - if (typeof item !== 'object' || item === null || Array.isArray(item)) continue; - if (Object.hasOwn(item, segment)) next.push((item as Record)[segment]); - } - } - current = next; - } - // The leaf spreads too, so a filter on `tags` matches a record whose - // `tags` array contains the value. - return current.flatMap(spreadValue); -}; - // An `undefined` field is a field the record does not have; storing the key // would make `'u' in content` differ from adapters that serialize to JSON. const withoutUndefined = (content: Record): Record => @@ -110,11 +84,13 @@ export class MemoryAdapter implements StackAdapter { filter: { content: 'path', contentPresent: true, + contentByType: 64, search: false, }, sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: true, + contentFieldByType: 64, }, limits: { attachmentBytes: null, @@ -347,39 +323,20 @@ export class MemoryAdapter implements StackAdapter { const relatedTo = f.relatedTo; results = results.filter((r) => matchesRelatedTo(r.associations, relatedTo)); } - // A `null` filter value means "field absent or null" — not "match - // nothing". Plain `===` would miss an absent field; treat both as - // satisfying a null filter (docs/spec/data-model.md § Filter). - if (f.content) { - const entries = Object.entries(f.content).map( - ([key, value]) => [parseContentFilterKey(key), value] as const, - ); - results = results.filter((r) => - entries.every(([segments, value]) => { - const found = valuesAtContentPath(r.content as Record, segments); - if (value === null || value === undefined) { - return found.length === 0 || found.some((v) => v === null || v === undefined); - } - return found.some((v) => v === value); - }), + if (f.content || f.contentPresent?.length) { + results = results.filter((r) => matchesContent(r.content as Record, f)); + } + if (f.contentByType) { + const arms = f.contentByType; + results = results.filter( + (r) => + Object.hasOwn(arms, r.typeId) && + matchesContent(r.content as Record, arms[r.typeId]), ); } results = this.sortRecords(results, query.sort); - // Element-wise like the content filter above: a path holds a value - // when at least one non-null value is reachable at it. - if (f.contentPresent?.length) { - const paths = f.contentPresent.map((key) => parseContentFilterKey(key)); - results = results.filter((r) => - paths.every((segments) => - valuesAtContentPath(r.content as Record, segments).some( - (v) => v !== null && v !== undefined, - ), - ), - ); - } - const limit = query.limit ?? 50; const start = query.cursor ? this.decodeCursor(query.cursor, query.sort) : 0; const page = results.slice(start, start + limit); @@ -391,9 +348,10 @@ export class MemoryAdapter implements StackAdapter { /** The sort a cursor was minted under, as an opaque comparable descriptor. */ private sortDescriptor(sort: QuerySort | undefined): string { + const content = sort?.contentField ?? sort?.contentFieldByType; return JSON.stringify([ - sort?.contentField ?? null, - sort?.contentField === undefined ? (sort?.field ?? 'createdAt') : null, + content ?? null, + content === undefined ? (sort?.field ?? 'createdAt') : null, sort ? requireDirection(sort) : 'desc', ]); } @@ -460,7 +418,7 @@ export class MemoryAdapter implements StackAdapter { const sign = direction === 'asc' ? 1 : -1; const byId = (a: StackRecord, b: StackRecord) => (a.id < b.id ? -sign : a.id > b.id ? sign : 0); - if (sort?.contentField === undefined) { + if (sort?.contentField === undefined && sort?.contentFieldByType === undefined) { const field = sort?.field ?? 'createdAt'; const value = (r: StackRecord) => field === 'version' @@ -469,8 +427,7 @@ export class MemoryAdapter implements StackAdapter { return [...records].sort((a, b) => sign * (value(a) - value(b)) || byId(a, b)); } - const field = sort.contentField; - const entries = new Map(records.map((r) => [r.id, this.sortEntry(r, field)])); + const entries = new Map(records.map((r) => [r.id, this.sortEntry(r, sort)])); return [...records].sort( (a, b) => compareSortEntries(entries.get(a.id)!, entries.get(b.id)!, direction) || byId(a, b), ); @@ -482,7 +439,16 @@ export class MemoryAdapter implements StackAdapter { * its type says it is, and an undeclared or non-scalar field orders as * nothing at all. */ - private sortEntry(record: StackRecord, field: string): SortEntry | null { + private sortEntry(record: StackRecord, sort: QuerySort): SortEntry | null { + let field = sort.contentField; + if (sort.contentFieldByType) { + const key = Object.hasOwn(sort.contentFieldByType, record.typeId) + ? sort.contentFieldByType[record.typeId] + : { value: null }; + if (key.field === undefined) return constantSortEntry(key.value ?? null); + field = key.field; + } + if (field === undefined) return null; const def = this.types.get(record.typeId)?.schema[field]; if (!def || def.kind === 'array' || def.kind === 'object') return null; return contentSortEntry(def.kind, (record.content as Record)[field]); @@ -696,11 +662,13 @@ export class IncapableMemoryAdapter extends MemoryAdapter { filter: { content: 'none', contentPresent: false, + contentByType: 0, search: false, }, sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: false, + contentFieldByType: 0, }, limits: { attachmentBytes: null, diff --git a/packages/core/src/type-handle.ts b/packages/core/src/type-handle.ts index 03243c3e..257f7a09 100644 --- a/packages/core/src/type-handle.ts +++ b/packages/core/src/type-handle.ts @@ -14,6 +14,8 @@ import { StackBadRequestError, StackMigrationError } from './errors.js'; import { lineageProblem, parseTypeId } from './schema.js'; import type { BaseId, ScalarFieldKind, TypeId } from './types/index.js'; +import { applySteps, fieldOrigins, recordOrigins } from './migration-steps.js'; +import type { MigrationStep } from './migration-steps.js'; // ------------------------------------------------------- // Schema literal types @@ -179,33 +181,48 @@ export const typeHandle = ({ /** * A step from one version of a type to another, passed to `Stack.open()` - * and typed on both versions' content. `migrate` is a method so a typed - * migration fits a list of untyped ones. See docs/spec/data-model.md § Type - * migrations. + * and typed on both versions' content. `steps` is what a collection reads + * to carry a content query across versions; `migrate` applies them, and is + * a method so a typed migration fits a list of untyped ones. See + * docs/spec/data-model.md § Type migrations. */ export type Migration, To = Record> = { readonly from: TypeId; readonly to: TypeId; + readonly steps: readonly MigrationStep[]; migrate(content: From): To; }; /** - * Build the migration from `from` to `to`, checking `fn` against both - * handles' stored content: a stored enum may hold a value either handle - * does not list, and a step carries it through. Within a family, `to` must name `from` as its - * `migratesFrom`; a step into another family has no lineage to check. See - * docs/spec/data-model.md § Type migrations. + * Build the migration from `from` to `to` out of declared steps, checked + * against both handles' schemas here: a step naming a field that is not + * there, a field `to` would refuse, or a required one nothing produces + * throws `StackMigrationError`. `derive` reads `from`'s stored content, in + * which an enum may hold a value neither handle lists. Within a family, + * `to` must name `from` as its `migratesFrom`; a step into another family + * has no lineage to check. See docs/spec/data-model.md § Migration steps. */ export const migration = ( from: TypeHandle, to: TypeHandle, - fn: (content: StoredContentOf) => NoInfer>, + steps: readonly MigrationStep>>[], ): Migration, StoredContentOf> => { + const refuse = (problem: string) => + new StackMigrationError(`Cannot migrate "${from.id}" to "${to.id}": ${problem}.`); if (to.baseId === from.baseId && to.migratesFrom !== from.id) { - throw new StackMigrationError( - `Cannot migrate "${from.id}" to "${to.id}": "${to.id}" migrates from ` + - `${to.migratesFrom ? `"${to.migratesFrom}"` : 'nothing'}.`, + throw refuse( + `"${to.id}" migrates from ${to.migratesFrom ? `"${to.migratesFrom}"` : 'nothing'}`, ); } - return Object.freeze({ from: from.id, to: to.id, migrate: fn }); + const frozen = Object.freeze([...steps]) as readonly MigrationStep[]; + const origins = fieldOrigins(frozen, from.schema, to.schema); + if (typeof origins === 'string') throw refuse(origins); + const built = Object.freeze({ + from: from.id, + to: to.id, + steps: frozen, + migrate: (content: Record) => applySteps(frozen, content), + }) as unknown as Migration, StoredContentOf>; + recordOrigins(built, origins); + return built; }; diff --git a/packages/core/src/types/capabilities.ts b/packages/core/src/types/capabilities.ts index d1f03a98..90f37098 100644 --- a/packages/core/src/types/capabilities.ts +++ b/packages/core/src/types/capabilities.ts @@ -39,6 +39,13 @@ export type StackCapabilities = { * one is there. Meaningful only where `content` is not `'none'`. */ contentPresent: boolean; + /** + * The most arms `filter.contentByType` may hold; `0` declines it. A + * collection narrows the versions it cannot reach rather than failing. + * Local adapters must honor it. See docs/spec/adapters.md § Adapter + * capabilities. + */ + contentByType: number; /** * Whether `filter.search` is honored. Not a rung on the ladder: a * full-text index is a different mechanism from field matching, and @@ -55,6 +62,8 @@ export type StackCapabilities = { * scalar. Independent of `filter.content`. */ contentField: boolean; + /** The most keys `sort.contentFieldByType` may hold; `0` declines it. */ + contentFieldByType: number; }; /** * Ceilings a client can check before spending a request; the server's @@ -83,6 +92,8 @@ export type StackCapabilities = { export type MissingCapability = | 'filter.content' | 'filter.contentPresent' + | 'filter.contentByType' | 'filter.search' | 'sort.fields' - | 'sort.contentField'; + | 'sort.contentField' + | 'sort.contentFieldByType'; diff --git a/packages/core/src/types/query.ts b/packages/core/src/types/query.ts index ce93666a..0ef461dd 100644 --- a/packages/core/src/types/query.ts +++ b/packages/core/src/types/query.ts @@ -39,6 +39,12 @@ export type DateRange = { after?: Date; }; +/** One Type's arm of `RecordFilter.contentByType`, flat by design. */ +export type ContentArm = { + content?: Record; + contentPresent?: string[]; +}; + export type RecordFilter = { // Native fields typeId?: TypeId | TypeId[]; @@ -93,6 +99,15 @@ export type RecordFilter = { */ contentPresent?: string[]; + /** + * A content predicate per Type: a record matches when its `typeId` is a + * key and its content satisfies that arm. Generated by a collection from + * a migration chain; never beside `content` or `contentPresent`. Needs + * `filter.contentByType`. POST /records/query only. See + * docs/spec/data-model.md § Content by type. + */ + contentByType?: Record; + // Full-text search (capability varies by adapter) search?: string; @@ -118,14 +133,40 @@ export const NATIVE_SORT_FIELDS = ['createdAt', 'updatedAt', 'version'] as const export type NativeSortField = (typeof NATIVE_SORT_FIELDS)[number]; /** - * Order by a native column or a top-level content field — two members, so - * a content field named `version` stays distinct from the native column. - * `direction` defaults to `asc`; a query with no sort is `createdAt`, - * newest first. See docs/spec/data-model.md § Sorting by a content field. + * What a record at one Type orders by under `contentFieldByType`: a + * top-level field of its own, or a constant already reduced to what it + * orders as — a number, text, or `null` for no value. + */ +export type ContentSortKey = + | { field: string; value?: never } + | { field?: never; value: string | number | null }; + +/** + * Order by a native column, a top-level content field, or a content field + * per Type — separate members, so a content field named `version` stays + * distinct from the native column. `direction` defaults to `asc`; a query + * with no sort is `createdAt`, newest first. See docs/spec/data-model.md + * § Sorting by a content field. */ export type QuerySort = - | { field: NativeSortField; contentField?: never; direction?: 'asc' | 'desc' } - | { field?: never; contentField: string; direction?: 'asc' | 'desc' }; + | { + field: NativeSortField; + contentField?: never; + contentFieldByType?: never; + direction?: 'asc' | 'desc'; + } + | { + field?: never; + contentField: string; + contentFieldByType?: never; + direction?: 'asc' | 'desc'; + } + | { + field?: never; + contentField?: never; + contentFieldByType: Record; + direction?: 'asc' | 'desc'; + }; export type StackQuery = { filter?: RecordFilter; diff --git a/packages/core/src/wire/request.ts b/packages/core/src/wire/request.ts index b7b8d50c..fedf971f 100644 --- a/packages/core/src/wire/request.ts +++ b/packages/core/src/wire/request.ts @@ -36,6 +36,8 @@ import type { VersionsQuery, NativeSortField, QuerySort, + ContentArm, + ContentSortKey, RecordFilter, RelatedToFilter, AttachmentFilter, @@ -103,6 +105,7 @@ const QUERY_BODY_FILTER_KEYS = [ 'relatedTo', 'content', 'contentPresent', + 'contentByType', 'search', 'includeDeleted', 'includeUnlisted', @@ -149,22 +152,32 @@ function requireSortField(raw: unknown): NativeSortField { } /** - * The two sort forms are mutually exclusive, and a request naming both is - * refused rather than resolved in one direction: a content field may be - * called `version`, so guessing which was meant is exactly the conflation - * the separate member exists to prevent. + * The sort forms are mutually exclusive, and a request naming more than + * one is refused rather than resolved in one direction: a content field may + * be called `version`, so guessing which was meant is exactly the + * conflation the separate members exist to prevent. */ function buildSort( field: unknown, contentField: unknown, direction: unknown, + contentFieldByType?: unknown, ): QuerySort | undefined { - if (field !== undefined && field !== null && contentField !== undefined && contentField !== null) + const named = [field, contentField, contentFieldByType].filter( + (member) => member !== undefined && member !== null, + ); + if (named.length > 1) throw new StackBadRequestError( - 'A sort names either a native field or a content field, never both.', + 'A sort names one of a native field, a content field or a content field per type.', ); const dir = direction === undefined || direction === null ? undefined : requireSortDirection(direction); + if (contentFieldByType !== undefined && contentFieldByType !== null) { + return { + contentFieldByType: parseSortByTypeBody(contentFieldByType), + ...(dir && { direction: dir }), + }; + } if (contentField !== undefined && contentField !== null) { return { contentField: requireString(contentField, 'sort content field'), @@ -281,7 +294,7 @@ const BASE_ID_REFUSAL = 'concrete typeId set it resolves to. See docs/spec/data-model.md § Filter.'; const PRESENT_AT_REFUSAL = - 'presentAt applies migration functions, which are app code rather than server code, so no ' + + 'presentAt applies migrations, which are app code rather than server code, so no ' + 'server can honor it — read records as stored and migrate them client-side. See ' + 'docs/spec/data-model.md § Type migrations.'; @@ -536,6 +549,44 @@ function parseLimitValue(raw: unknown): number { return raw; } +/** + * `filter.contentByType` as the body carries it: one flat arm per key. + * What the keys and arms may hold beyond their shape — a TypeId, at least + * one arm, never beside `content` — is assertValidContentByType()'s, which + * `Stack.query()` applies to every caller. See docs/spec/data-model.md + * § Content by type. + */ +function parseContentByTypeBody(raw: unknown): Record { + const arms: Record = {}; + for (const [typeId, value] of Object.entries(requirePlainObject(raw, 'filter.contentByType'))) { + const label = `filter.contentByType["${typeId}"]`; + const arm = requireKnownKeys(value, ['content', 'contentPresent'], label); + arms[typeId] = { + ...(arm.content !== undefined && { + content: requirePlainObject(arm.content, `${label}.content`), + }), + ...(arm.contentPresent !== undefined && { + contentPresent: requireStringArray(arm.contentPresent, `${label}.contentPresent`), + }), + }; + } + return arms; +} + +/** `sort.contentFieldByType` as the body carries it; see assertValidSort() for the rest. */ +function parseSortByTypeBody(raw: unknown): Record { + const keys: Record = {}; + for (const [typeId, value] of Object.entries( + requirePlainObject(raw, 'sort.contentFieldByType'), + )) { + const label = `sort.contentFieldByType["${typeId}"]`; + // Copied whole, so a key naming both a field and a value reaches the + // check that refuses it rather than losing one half here. + keys[typeId] = { ...requireKnownKeys(value, ['field', 'value'], label) } as ContentSortKey; + } + return keys; +} + /** * Build a `StackQuery` from a `POST /records/query` body — the superset * form, which also carries `filter.content`, with ISO date strings decoded. @@ -578,6 +629,8 @@ export function parseQueryBody(raw: unknown): StackQuery { if (f.content !== undefined) filter.content = requirePlainObject(f.content, 'filter.content'); if (f.contentPresent !== undefined) filter.contentPresent = requireStringArray(f.contentPresent, 'filter.contentPresent'); + if (f.contentByType !== undefined) + filter.contentByType = parseContentByTypeBody(f.contentByType); if (f.search !== undefined) filter.search = requireString(f.search, 'filter.search'); if (optionalBoolean(f.includeDeleted, 'filter.includeDeleted')) filter.includeDeleted = true; if (optionalBoolean(f.includeUnlisted, 'filter.includeUnlisted')) filter.includeUnlisted = true; @@ -601,8 +654,12 @@ export function parseQueryBody(raw: unknown): StackQuery { } if (body.sort !== undefined) { - const s = requireKnownKeys(body.sort, ['field', 'contentField', 'direction'], 'sort'); - const sort = buildSort(s.field, s.contentField, s.direction); + const s = requireKnownKeys( + body.sort, + ['field', 'contentField', 'contentFieldByType', 'direction'], + 'sort', + ); + const sort = buildSort(s.field, s.contentField, s.direction, s.contentFieldByType); if (!sort) throw new StackBadRequestError('Invalid sort: expected a field or a contentField.'); query.sort = sort; } diff --git a/packages/core/tests/change-events.test.ts b/packages/core/tests/change-events.test.ts index 05e29823..888aa2c5 100644 --- a/packages/core/tests/change-events.test.ts +++ b/packages/core/tests/change-events.test.ts @@ -249,7 +249,7 @@ describe('every record emits, including the ones a query hides', () => { schema: NoteV1.schema, migratesFrom: NoteV1, }); - stack = await Stack.open(adapter, { migrations: [migration(NoteV1, NoteV2, (c) => c)] }); + stack = await Stack.open(adapter, { migrations: [migration(NoteV1, NoteV2, [])] }); await stack.defineType(NoteV2); const ids = []; for (let i = 0; i < 3; i++) ids.push((await stack.create(NOTE, { text: `n${i}` })).id); diff --git a/packages/core/tests/collection.test.ts b/packages/core/tests/collection.test.ts index e9c45a26..0085c9ff 100644 --- a/packages/core/tests/collection.test.ts +++ b/packages/core/tests/collection.test.ts @@ -2,6 +2,7 @@ import { describe, test, expect, expectTypeOf, beforeEach, vi } from 'vitest'; import { Stack } from '../src/stack/stack.js'; import { MemoryAdapter } from '../src/testing.js'; import { migration, typeHandle } from '../src/type-handle.js'; +import { add, derive, drop, map, rename } from '../src/migration-steps.js'; import type { ContentOf, ReadonlyTypeSchema, UnlistedValue } from '../src/type-handle.js'; import { MisfitError } from '../src/collection.js'; import type { @@ -64,10 +65,7 @@ const Shelf = typeHandle({ schema: { name: { kind: 'string', required: true } }, }); -const bookV1ToV2 = migration(BookV1, Book, (c) => ({ - ...c, - status: c.status === 'done' ? 'finished' : c.status, -})); +const bookV1ToV2 = migration(BookV1, Book, [map('status', { done: 'finished' })]); // The naming an app does: an interface keeps its name where a type alias would not. // eslint-disable-next-line @typescript-eslint/no-empty-object-type @@ -269,35 +267,92 @@ describe.each([ expect(records.map((r) => r.content.title)).toEqual(['Emma']); }); - test("a content filter reaches only the handle's version and flags both sides", async () => { + test('a content filter reaches older versions through a map, and flags the newer side', async () => { const book = await books.create({ title: 'Dune', status: 'finished' }); - await createV1('Emma', 'done'); + const old = await createV1('Emma', 'done'); + await createV1('Hamlet', 'want'); await stack.create(BookV3.id, { title: 'Ulysses', status: 'finished' }); const page = await books.query({ filter: { content: { status: 'finished' } } }); - expect(page.records.map((r) => r.id)).toEqual([book.id]); + expect(page.records.map((r) => r.id).sort()).toEqual([book.id, old.id].sort()); + expect(page.records.find((r) => r.id === old.id)?.content.status).toBe('finished'); expect(page.misfits).toEqual([]); - expect(page.skipped).toEqual({ older: true, newer: true }); + expect(page.skipped).toEqual({ older: false, newer: true }); + }); + + test('an older record holding the value a map also produces is flagged, not matched', async () => { + // An enum widens in place, so @1 can come to hold a value its handle + // does not list. + await stack.defineType({ + ...BookV1, + schema: { + ...BookV1.schema, + status: { kind: 'enum', values: ['want', 'reading', 'done', 'finished'], required: true }, + }, + }); + await createV1('Emma', 'finished'); + const page = await books.query({ filter: { content: { status: 'finished' } } }); + expect(page.records).toEqual([]); + expect(page.skipped).toEqual({ older: true, newer: false }); + }); + + test('a value no older value maps to drops the older arm without a skip', async () => { + await createV1('Emma', 'done'); + // A value the handle does not list is filtered for with a cast. + const page = await books.query({ filter: { content: { status: 'done' as never } } }); + expect(page.records).toEqual([]); + expect(page.skipped).toEqual({ older: false, newer: false }); + }); + + test('a field no step produces is no value at an older version', async () => { + const book = await books.create({ title: 'Dune', status: 'want', pages: 412 }); + const old = await createV1(); + const absent = await books.query({ filter: { content: { pages: null } } }); + expect(absent.records.map((r) => r.id)).toEqual([old.id]); + expect(absent.skipped).toEqual({ older: false, newer: false }); + const present = await books.query({ filter: { contentPresent: ['pages'] } }); + expect(present.records.map((r) => r.id)).toEqual([book.id]); + expect(present.skipped).toEqual({ older: false, newer: false }); + }); + + test('an undefined content value is a filter, as an adapter reads it', async () => { + await books.create({ title: 'Dune', status: 'want' }); + await createV1(); + const page = await books.query({ filter: { content: { pages: undefined } } }); + expect(page.records).toHaveLength(2); + expect(page.skipped).toEqual({ older: false, newer: false }); + }); + + test('a content sort orders records at every version it reaches as one listing', async () => { + await books.create({ title: 'Dune', status: 'want' }); + await createV1('Anathem'); + await books.create({ title: 'Middlemarch', status: 'want' }); + await createV1('Emma'); + const page = await books.query({ sort: { contentField: 'title' } }); + expect(page.records.map((r) => r.content.title)).toEqual([ + 'Anathem', + 'Dune', + 'Emma', + 'Middlemarch', + ]); + expect(page.skipped).toEqual({ older: false, newer: false }); }); - test('a content sort and a presence filter narrow the same way', async () => { + test('a sort on a mapped field narrows to the versions that keep its order', async () => { const book = await books.create({ title: 'Dune', status: 'want' }); await createV1(); - const sorted = await books.query({ sort: { contentField: 'title' } }); - expect(sorted.records.map((r) => r.id)).toEqual([book.id]); - expect(sorted.skipped).toEqual({ older: true, newer: false }); - const present = await books.query({ filter: { contentPresent: ['title'] } }); - expect(present.records.map((r) => r.id)).toEqual([book.id]); - expect(present.skipped).toEqual({ older: true, newer: false }); + const page = await books.query({ sort: { contentField: 'status' } }); + expect(page.records.map((r) => r.id)).toEqual([book.id]); + expect(page.skipped).toEqual({ older: true, newer: false }); }); test('skipped is reported on the first page only', async () => { await books.create({ title: 'Dune', status: 'want' }); await books.create({ title: 'Emma', status: 'want' }); await createV1(); - const first = await books.query({ filter: { content: { status: 'want' } }, limit: 1 }); + const first = await books.query({ sort: { contentField: 'status' }, limit: 1 }); expect(first.skipped).toEqual({ older: true, newer: false }); const second = await books.query({ - filter: { content: { status: 'want' } }, + sort: { contentField: 'status' }, limit: 1, cursor: first.cursor!, }); @@ -309,7 +364,10 @@ describe.each([ await books.create({ title: 'Dune', status: 'want' }); await createV1(); await stack.create(BookV3.id, { title: 'Ulysses', status: 'want' }); - const page = await books.query({ filter: { content: { status: 'want' }, tags: ['fantasy'] } }); + const page = await books.query({ + filter: { tags: ['fantasy'] }, + sort: { contentField: 'status' }, + }); expect(page.skipped).toEqual({ older: false, newer: false }); }); @@ -321,14 +379,6 @@ describe.each([ expect(page.skipped).toEqual({ older: false, newer: false }); }); - test('an undefined content value is a filter, as an adapter reads it', async () => { - await books.create({ title: 'Dune', status: 'want' }); - await createV1(); - const page = await books.query({ filter: { content: { pages: undefined } } }); - expect(page.records).toHaveLength(1); - expect(page.skipped).toEqual({ older: true, newer: false }); - }); - test('query() drops typeId and presentAt from an untyped caller', async () => { const book = await books.create({ title: 'Dune', status: 'want' }); const newer = await stack.create(BookV3.id, { title: 'Emma', status: 'want' }); @@ -349,7 +399,7 @@ describe.each([ test('a read of an older record with no migration to bridge the gap throws', async () => { const reopened = await Stack.open(adapter, { - migrations: [migration(Book, BookV3, (c) => c)], + migrations: [migration(Book, BookV3, [])], }); const old = await createV1(); const v3 = view(reopened).collection(BookV3); @@ -542,7 +592,10 @@ describe.each([ beforeEach(async () => { const adapter = await MemoryAdapter.open({ ownerEntityId: OWNER, timezone: 'UTC' }); stack = await Stack.open(adapter, { - migrations: [bookV1ToV2, migration(Shelf, Book, (c) => ({ title: c.name, status: 'want' }))], + migrations: [ + bookV1ToV2, + migration(Shelf, Book, [rename('name', 'title'), add('status', 'want')]), + ], }); await stack.defineType(BookV1); await stack.defineType(Book); @@ -579,15 +632,192 @@ describe.each([ }); }); -describe('a collection whose migration function throws', () => { +const TaskV1 = typeHandle({ + id: 'com.example.tasks/task@1', + name: 'Task', + schema: { + author: { kind: 'string', required: true }, + status: { kind: 'enum', values: ['open', 'done'], required: true }, + legacy: { kind: 'boolean' }, + }, +}); + +const TaskV2 = typeHandle({ + id: 'com.example.tasks/task@2', + name: 'Task', + migratesFrom: TaskV1, + schema: { + by: { kind: 'string', required: true }, + status: { kind: 'enum', values: ['open', 'finished'], required: true }, + priority: { kind: 'enum', values: ['high', 'low', 'normal'], required: true }, + slug: { kind: 'string' }, + }, +}); + +const Task = typeHandle({ + id: 'com.example.tasks/task@3', + name: 'Task', + migratesFrom: TaskV2, + schema: { + owner: { kind: 'string', required: true }, + status: TaskV2.schema.status, + priority: TaskV2.schema.priority, + slug: TaskV2.schema.slug, + }, +}); + +const TASK_MIGRATIONS = [ + migration(TaskV1, TaskV2, [ + rename('author', 'by'), + map('status', { done: 'finished' }), + add('priority', 'normal'), + drop('legacy'), + derive('slug', (c) => c.author.toLowerCase()), + ]), + migration(TaskV2, Task, [rename('by', 'owner')]), +]; + +describe.each([ + ['Stack', (s: Stack): StackClient => s], + ['ScopedStack', (s: Stack): StackClient => s.asEntity(OWNER)], +] as const)('content queries across a migration chain, on %s', (_name, view) => { + let adapter: MemoryAdapter; + let stack: Stack; + + const open = async (filterArms = 64, sortKeys = 64) => { + adapter = await MemoryAdapter.open({ ownerEntityId: OWNER, timezone: 'UTC' }); + (adapter as { capabilities: StackClient['capabilities'] }).capabilities = { + ...adapter.capabilities, + filter: { ...adapter.capabilities.filter, contentByType: filterArms }, + sort: { ...adapter.capabilities.sort, contentFieldByType: sortKeys }, + }; + stack = await Stack.open(adapter, { migrations: TASK_MIGRATIONS }); + for (const type of [TaskV1, TaskV2, Task]) await stack.defineType(type); + }; + + /** One task at each version, all owned by `owner`. */ + const seed = async (owner: string) => [ + await stack.create(TaskV1.id, { author: owner, status: 'open' }), + await stack.create(TaskV2.id, { by: owner, status: 'open', priority: 'high', slug: 'x' }), + await stack.create(Task.id, { owner, status: 'open', priority: 'low' }), + ]; + + const tasks = () => view(stack).collection(Task); + + beforeEach(() => open()); + + test('a filter on a field renamed across two steps reaches every version', async () => { + const ada = await seed('Ada'); + await seed('Grace'); + const page = await tasks().query({ filter: { content: { owner: 'Ada' } } }); + expect(page.records.map((r) => r.id).sort()).toEqual(ada.map((r) => r.id).sort()); + expect(page.records.map((r) => r.typeId)).toEqual([Task.id, Task.id, Task.id]); + expect(page.skipped).toEqual({ older: false, newer: false }); + }); + + test('a sort on a renamed field orders every version as one listing', async () => { + await stack.create(TaskV1.id, { author: 'Cleo', status: 'open' }); + await stack.create(Task.id, { owner: 'Ada', status: 'open', priority: 'low' }); + await stack.create(TaskV2.id, { by: 'Bea', status: 'open', priority: 'low' }); + const page = await tasks().query({ sort: { contentField: 'owner', direction: 'desc' } }); + expect(page.records.map((r) => r.content.owner)).toEqual(['Cleo', 'Bea', 'Ada']); + }); + + test('a filter on an added field is decided in core, keeping or dropping the older arm', async () => { + const [v1] = await seed('Ada'); + const normal = await tasks().query({ filter: { content: { priority: 'normal' } } }); + expect(normal.records.map((r) => r.id)).toEqual([v1.id]); + const high = await tasks().query({ filter: { content: { priority: 'high' } } }); + expect(high.records.map((r) => r.content.priority)).toEqual(['high']); + expect(high.skipped).toEqual({ older: false, newer: false }); + }); + + test('a sort on an added field orders the older records as the constant', async () => { + await seed('Ada'); + const page = await tasks().query({ sort: { contentField: 'priority' } }); + expect(page.records.map((r) => [r.content.priority, r.content.owner])).toEqual([ + ['high', 'Ada'], + ['low', 'Ada'], + ['normal', 'Ada'], + ]); + }); + + test('a filter on a derived field narrows the versions before the derivation', async () => { + const [, v2] = await seed('Ada'); + const page = await tasks().query({ filter: { content: { slug: 'x' } } }); + expect(page.records.map((r) => r.id)).toEqual([v2.id]); + expect(page.skipped).toEqual({ older: true, newer: false }); + }); + + test('pages through every version under a translated sort', async () => { + for (const owner of ['Fay', 'Ada', 'Eve', 'Bea', 'Dot', 'Cleo']) await seed(owner); + const owners: string[] = []; + let cursor: string | undefined; + do { + const page = await tasks().query({ + filter: { content: { status: 'open' } }, + sort: { contentField: 'owner' }, + limit: 4, + ...(cursor && { cursor }), + }); + owners.push(...page.records.map((r) => r.content.owner)); + cursor = page.cursor ?? undefined; + } while (cursor); + expect(owners).toEqual( + ['Ada', 'Bea', 'Cleo', 'Dot', 'Eve', 'Fay'].flatMap((owner) => [owner, owner, owner]), + ); + }); + + test('an adapter that declines the shape gets the handle-only query', async () => { + await open(0, 0); + const [, , v3] = await seed('Ada'); + const filtered = await tasks().query({ filter: { content: { owner: 'Ada' } } }); + expect(filtered.records.map((r) => r.id)).toEqual([v3.id]); + expect(filtered.skipped).toEqual({ older: true, newer: false }); + const sorted = await tasks().query({ sort: { contentField: 'owner' } }); + expect(sorted.records.map((r) => r.id)).toEqual([v3.id]); + expect(sorted.skipped).toEqual({ older: true, newer: false }); + }); + + test('an adapter with a lower cap gets the newest versions that fit', async () => { + await open(2); + const [, v2, v3] = await seed('Ada'); + const page = await tasks().query({ filter: { content: { owner: 'Ada' } } }); + expect(page.records.map((r) => r.id).sort()).toEqual([v2.id, v3.id].sort()); + expect(page.skipped).toEqual({ older: true, newer: false }); + }); + + test('a migration not built by migration() is narrowed', async () => { + stack = await Stack.open(adapter, { + migrations: [ + TASK_MIGRATIONS[0], + { + from: TaskV2.id, + to: Task.id, + steps: [], + migrate: (c: Record) => TASK_MIGRATIONS[1].migrate(c as never), + }, + ], + }); + const [, , v3] = await seed('Ada'); + const page = await tasks().query({ filter: { content: { owner: 'Ada' } } }); + expect(page.records.map((r) => r.id)).toEqual([v3.id]); + expect(page.skipped).toEqual({ older: true, newer: false }); + }); +}); + +describe('a collection whose migration throws', () => { test('subscribe() still delivers the change, without its record', async () => { const stack = await Stack.open( await MemoryAdapter.open({ ownerEntityId: OWNER, timezone: 'UTC' }), { migrations: [ - migration(BookV1, Book, () => { - throw new TypeError('broken migration'); - }), + migration(BookV1, Book, [ + map('status', { done: 'finished' }), + derive('title', () => { + throw new TypeError('broken migration'); + }), + ]), ], }, ); diff --git a/packages/core/tests/combine.test.ts b/packages/core/tests/combine.test.ts index ea8ab05b..be89b2c7 100644 --- a/packages/core/tests/combine.test.ts +++ b/packages/core/tests/combine.test.ts @@ -29,11 +29,13 @@ const capabilities: StackCapabilities = { filter: { content: 'none', contentPresent: false, + contentByType: 0, search: false, }, sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: false, + contentFieldByType: 0, }, limits: { attachmentBytes: null, diff --git a/packages/core/tests/install.test.ts b/packages/core/tests/install.test.ts index fd4120d4..b25ff6b8 100644 --- a/packages/core/tests/install.test.ts +++ b/packages/core/tests/install.test.ts @@ -10,6 +10,7 @@ import { import { MemoryAdapter } from '../src/testing.js'; import { isPlanEmpty } from '../src/install.js'; import { migration, typeHandle } from '../src/type-handle.js'; +import { add } from '../src/migration-steps.js'; import type { AppManifest } from '../src/install.js'; import type { AppContent, GrantContent, InstallContent, StackRecord } from '../src/types/index.js'; @@ -608,7 +609,7 @@ describe('commitMigration() for an installed app', () => { describe("migrateAll({ sweep: 'listed' })", () => { test('migrates live, listed records and counts the deleted ones it passed over', async () => { stack = await Stack.open(await MemoryAdapter.open({ ownerEntityId: OWNER }), { - migrations: [migration(NOTE_1_TYPE, NOTE_2_TYPE, (c) => ({ ...c, pinned: false }))], + migrations: [migration(NOTE_1_TYPE, NOTE_2_TYPE, [add('pinned', false)])], }); await stack.defineType(NOTE_1_TYPE); await stack.defineType(NOTE_2_TYPE); diff --git a/packages/core/tests/migration-steps.test.ts b/packages/core/tests/migration-steps.test.ts new file mode 100644 index 00000000..5d0be638 --- /dev/null +++ b/packages/core/tests/migration-steps.test.ts @@ -0,0 +1,86 @@ +import { describe, test, expect } from 'vitest'; +import { add, composeOrigins, derive, fieldOrigins, map, rename } from '../src/migration-steps.js'; +import type { FieldOrigins } from '../src/migration-steps.js'; +import type { ReadonlyTypeSchema } from '../src/type-handle.js'; + +const origins = ( + steps: Parameters[0], + from: ReadonlyTypeSchema, + to: ReadonlyTypeSchema, +): FieldOrigins => { + const result = fieldOrigins(steps, from, to); + if (typeof result === 'string') throw new Error(result); + return result; +}; + +const status = (values: string[]) => ({ kind: 'enum', values }) as const; + +describe('fieldOrigins()', () => { + test("names each target field's origin after the steps run in order", () => { + const result = origins( + [rename('a', 'b'), map('b', { x: 'y' }), add('c', 'k'), derive('d', () => 1)], + { a: status(['x', 'z']) }, + { b: status(['y', 'z']), c: { kind: 'string' }, d: { kind: 'number' }, e: { kind: 'date' } }, + ); + expect(Object.fromEntries(result)).toEqual({ + b: { kind: 'field', field: 'a', values: { x: 'y' } }, + c: { kind: 'constant', value: 'k' }, + d: { kind: 'derived' }, + e: { kind: 'constant', value: undefined }, + }); + }); + + test('composes two maps of one field into one table', () => { + const result = origins( + [map('s', { a: 'b' }), map('s', { b: 'c', d: 'e' })], + { s: status(['a', 'b', 'd']) }, + { s: status(['c', 'e']) }, + ); + expect(result.get('s')).toEqual({ + kind: 'field', + field: 's', + values: { a: 'c', b: 'c', d: 'e' }, + }); + }); + + test('maps an added constant', () => { + const result = origins([add('s', 'a'), map('s', { a: 'b' })], {}, { s: status(['b']) }); + expect(result.get('s')).toEqual({ kind: 'constant', value: 'b' }); + }); +}); + +describe('composeOrigins()', () => { + test('follows a field through a rename at each step, composing its maps', () => { + const lower = origins( + [rename('a', 'b'), map('b', { x: 'y' })], + { a: status(['x']) }, + { b: status(['y']) }, + ); + const upper = origins( + [rename('b', 'c'), map('c', { y: 'z' })], + { b: status(['y']) }, + { c: status(['z']) }, + ); + expect(composeOrigins(lower, upper).get('c')).toEqual({ + kind: 'field', + field: 'a', + values: { x: 'z', y: 'z' }, + }); + }); + + test('carries a constant or a derivation from the lower step through the upper', () => { + const lower = origins( + [add('k', 'a'), derive('d', () => 1)], + {}, + { k: status(['a']), d: { kind: 'number' } }, + ); + const upper = origins( + [map('k', { a: 'b' })], + { k: status(['a']), d: { kind: 'number' } }, + { k: status(['b']), d: { kind: 'number' } }, + ); + const composed = composeOrigins(lower, upper); + expect(composed.get('k')).toEqual({ kind: 'constant', value: 'b' }); + expect(composed.get('d')).toEqual({ kind: 'derived' }); + }); +}); diff --git a/packages/core/tests/query-capabilities.test.ts b/packages/core/tests/query-capabilities.test.ts index 966fb69c..69cfea36 100644 --- a/packages/core/tests/query-capabilities.test.ts +++ b/packages/core/tests/query-capabilities.test.ts @@ -10,13 +10,27 @@ * send a caller to look at a discovery entry that is not the problem. */ import { describe, test, expect } from 'vitest'; -import { assertQueryCapabilities, assertSortCapability } from '../src/query-validation.js'; +import { + assertQueryCapabilities, + assertSortCapability, + assertValidContentByType, + assertValidSort, +} from '../src/query-validation.js'; import { StackBadRequestError } from '../src/errors.js'; -import type { MissingCapability, RecordFilter, StackCapabilities } from '../src/types/index.js'; +import type { + MissingCapability, + QuerySort, + RecordFilter, + StackCapabilities, +} from '../src/types/index.js'; const ALL: StackCapabilities = { - filter: { content: 'path', contentPresent: true, search: true }, - sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: true }, + filter: { content: 'path', contentPresent: true, contentByType: 64, search: true }, + sort: { + fields: ['createdAt', 'updatedAt', 'version'], + contentField: true, + contentFieldByType: 64, + }, limits: { attachmentBytes: null, contentBytes: null }, }; @@ -90,6 +104,63 @@ describe('a refused query names the capability it needed', () => { ).toBe('sort.contentField'); }); + const ARMS: RecordFilter = { + contentByType: { 'com.example/x@1': { content: { a: 1 } }, 'com.example/x@2': {} }, + }; + + test('filter.contentByType declined', () => { + expect( + refusalFor(() => assertQueryCapabilities(ARMS, without({ filter: { contentByType: 0 } }))), + ).toBe('filter.contentByType'); + }); + + test('filter.contentByType past its cap', () => { + expect( + refusalFor(() => assertQueryCapabilities(ARMS, without({ filter: { contentByType: 1 } }))), + ).toBe('filter.contentByType'); + expect(() => + assertQueryCapabilities(ARMS, without({ filter: { contentByType: 2 } })), + ).not.toThrow(); + }); + + test("filter.contentByType against reach 'none'", () => { + expect( + refusalFor(() => assertQueryCapabilities(ARMS, without({ filter: { content: 'none' } }))), + ).toBe('filter.content'); + }); + + test("an arm's keys are gated as top-level keys are", () => { + const arm = (predicate: object): RecordFilter => ({ + contentByType: { 'com.example/x@1': predicate }, + }); + expect( + refusalFor(() => + assertQueryCapabilities( + arm({ content: { 'a.b': 1 } }), + without({ filter: { content: 'field' } }), + ), + ), + ).toBe('filter.content'); + expect( + refusalFor(() => + assertQueryCapabilities( + arm({ contentPresent: ['a'] }), + without({ filter: { contentPresent: false } }), + ), + ), + ).toBe('filter.contentPresent'); + }); + + test('sort.contentFieldByType declined, or past its cap', () => { + const sort = { contentFieldByType: { 'com.example/x@1': { field: 'a' } } }; + expect( + refusalFor(() => assertSortCapability(sort, without({ sort: { contentFieldByType: 0 } }))), + ).toBe('sort.contentFieldByType'); + expect(() => + assertSortCapability(sort, without({ sort: { contentFieldByType: 1 } })), + ).not.toThrow(); + }); + test('sort.fields', () => { expect( refusalFor(() => @@ -111,3 +182,68 @@ describe('a query refused for its own shape names no capability', () => { ).toBeUndefined(); }); }); + +describe('the per-Type shapes are held to a split along Types', () => { + test.each([ + ['an empty map', { contentByType: {} }], + ['an arm keyed by something other than a TypeId', { contentByType: { note: {} } }], + ['an arm carrying another key', { contentByType: { 'com.example/x@1': { typeId: 'y' } } }], + ['an arm that is not an object', { contentByType: { 'com.example/x@1': [] } }], + [ + 'arms beside a top-level content filter', + { content: { a: 1 }, contentByType: { 'com.example/x@1': {} } }, + ], + [ + 'arms beside a top-level presence filter', + { contentPresent: ['a'], contentByType: { 'com.example/x@1': {} } }, + ], + [ + 'a malformed path in an arm', + { contentByType: { 'com.example/x@1': { content: { 'a..b': 1 } } } }, + ], + ])('filter.contentByType refuses %s', (_label, filter) => { + expect(refusalFor(() => assertValidContentByType(filter as RecordFilter))).toBeUndefined(); + }); + + test.each([ + ['an empty map', {}], + ['a key that is not a TypeId', { note: { field: 'a' } }], + ['a key naming both a field and a value', { 'com.example/x@1': { field: 'a', value: 1 } }], + ['a nested field', { 'com.example/x@1': { field: 'a.b' } }], + ['a value that is not a scalar', { 'com.example/x@1': { value: [] } }], + ['a value that is not finite', { 'com.example/x@1': { value: Infinity } }], + ])('sort.contentFieldByType refuses %s', (_label, contentFieldByType) => { + expect(refusalFor(() => assertValidSort({ contentFieldByType } as QuerySort))).toBeUndefined(); + }); + + test('a sort names one member only', () => { + expect( + refusalFor(() => + assertValidSort({ + contentField: 'a', + contentFieldByType: { 'com.example/x@1': { field: 'a' } }, + } as unknown as QuerySort), + ), + ).toBeUndefined(); + }); + + test('accepts a well-formed split', () => { + expect(() => + assertValidContentByType({ + contentByType: { + 'com.example/x@1': { content: { 'a.b': 1 }, contentPresent: ['c'] }, + 'com.example/x@2': {}, + }, + }), + ).not.toThrow(); + expect(() => + assertValidSort({ + contentFieldByType: { + 'com.example/x@1': { field: 'a' }, + 'com.example/x@2': { value: 3 }, + 'com.example/x@3': { value: null }, + }, + }), + ).not.toThrow(); + }); +}); diff --git a/packages/core/tests/stack.test.ts b/packages/core/tests/stack.test.ts index c4322790..b7dce379 100644 --- a/packages/core/tests/stack.test.ts +++ b/packages/core/tests/stack.test.ts @@ -24,6 +24,7 @@ import { } from '../src/id.js'; import type { TypeSchema } from '../src/types/index.js'; import { migration, typeHandle } from '../src/type-handle.js'; +import { add, derive, drop } from '../src/migration-steps.js'; import type { Migration } from '../src/type-handle.js'; import { RESERVED_CONTENT_KEYS, CONTENT_KEY_PATH_METACHARACTERS } from '../src/validate.js'; import { InvalidDidError } from '../src/did/did.js'; @@ -74,7 +75,7 @@ const NoteV2 = typeHandle({ schema: { text: { kind: 'text', required: true }, title: { kind: 'string' } }, migratesFrom: NoteV1, }); -const noteV1ToV2 = migration(NoteV1, NoteV2, (c) => ({ ...c, title: '' })); +const noteV1ToV2 = migration(NoteV1, NoteV2, [add('title', '')]); const EntityV1 = typeHandle({ id: '_entity@1', @@ -232,7 +233,7 @@ describe('Stack.open', () => { }); const s = await Stack.open(emptyAdapter, { ownerProfile: { name: 'Jane Smith' }, - migrations: [migration(EntityV1, EntityV2, (c) => ({ ...c }))], + migrations: [migration(EntityV1, EntityV2, [])], }); await s.defineType(EntityV2); await s.migrateAll('_entity'); @@ -305,7 +306,7 @@ describe('Stack.open', () => { }, migratesFrom: EntityV1, }); - await reopenWith(migration(EntityV1, EntityV2, (c) => ({ ...c }))); + await reopenWith(migration(EntityV1, EntityV2, [])); await stack.defineType(EntityV2); const created = await stack.create('_entity@1', { did: 'did:key:x', name: 'X' }); await stack.migrateAll('_entity'); @@ -2079,10 +2080,7 @@ describe("presentAt: 'latest' (explicit in-memory migration)", () => { }, migratesFrom: NoteV2, }); - await reopenWith( - noteV1ToV2, - migration(NoteV2, NoteV3, (content) => ({ ...content, pinned: false })), - ); + await reopenWith(noteV1ToV2, migration(NoteV2, NoteV3, [add('pinned', false)])); await stack.defineType(NoteV3); const record = await stack.create(NOTE_V1, { text: 'hello' }); @@ -2099,7 +2097,7 @@ describe("presentAt: 'latest' (explicit in-memory migration)", () => { describe('Stack.open({ migrations })', () => { test('refuses two migrations from the same TypeId', async () => { - const again = migration(NoteV1, NoteV2, (c) => ({ ...c, title: 'again' })); + const again = migration(NoteV1, NoteV2, [add('title', 'again')]); await expect(Stack.open(adapter, { migrations: [noteV1ToV2, again] })).rejects.toThrow( StackMigrationError, ); @@ -2111,9 +2109,9 @@ describe('Stack.open({ migrations })', () => { [NOTE_V1, 'com.example.test/note'], ['note', NOTE_V2], ]) { - await expect(Stack.open(adapter, { migrations: [{ from, to, migrate }] })).rejects.toThrow( - StackMigrationError, - ); + await expect( + Stack.open(adapter, { migrations: [{ from, to, steps: [], migrate }] }), + ).rejects.toThrow(StackMigrationError); } }); @@ -2121,7 +2119,7 @@ describe('Stack.open({ migrations })', () => { const migrate = (c: Record) => c; for (const to of [NOTE_V1, 'com.example.test/note@0']) { await expect( - Stack.open(adapter, { migrations: [{ from: NOTE_V1, to, migrate }] }), + Stack.open(adapter, { migrations: [{ from: NOTE_V1, to, steps: [], migrate }] }), ).rejects.toThrow(StackMigrationError); } }); @@ -2132,9 +2130,9 @@ describe('Stack.open({ migrations })', () => { await expect( Stack.open(adapter, { migrations: [ - { from: NOTE_V1, to: NOTE_V2, migrate }, - { from: NOTE_V2, to: other, migrate }, - { from: other, to: NOTE_V1, migrate }, + { from: NOTE_V1, to: NOTE_V2, steps: [], migrate }, + { from: NOTE_V2, to: other, steps: [], migrate }, + { from: other, to: NOTE_V1, steps: [], migrate }, ], }), ).rejects.toThrow(`The migrations passed form a cycle through "${NOTE_V1}".`); @@ -2144,7 +2142,7 @@ describe('Stack.open({ migrations })', () => { const Other = typeHandle({ id: 'com.example.other/note@1', name: 'Other', schema: {} }); await stack.defineType(Other); const note = await stack.create(NOTE_V1, { text: 'hi' }); - await reopenWith(migration(NoteV1, Other, () => ({}))); + await reopenWith(migration(NoteV1, Other, [drop('text')])); await stack.migrateAll(fam(NOTE_V1)); expect((await stack.get(note.id))?.typeId).toBe(Other.id); }); @@ -2184,10 +2182,7 @@ describe('migrateAll', () => { schema: NoteV2.schema, migratesFrom: NoteV2, }); - await reopenWith( - noteV1ToV2, - migration(NoteV2, NoteV3, (c) => ({ ...c })), - ); + await reopenWith(noteV1ToV2, migration(NoteV2, NoteV3, [])); await stack.defineType(NoteV3); const v1 = await stack.create(NOTE_V1, { text: 'one' }); const v2 = await stack.create(NOTE_V2, { text: 'two', title: 't' }); @@ -2239,15 +2234,14 @@ describe('migrateAll', () => { expect(versions[0].typeId).toBe(NOTE_V1); }); - test('aborts immediately if a migration function produces invalid content, leaving that record unmigrated', async () => { + test('aborts immediately if a migration produces invalid content, leaving that record unmigrated', async () => { // Fresh stack so this test can pass its own (deliberately buggy) // migration instead of the valid one from the outer beforeEach. const buggyAdapter = await MemoryAdapter.open({ ownerEntityId: 'owner-123', timezone: 'UTC' }); const buggyStack = await Stack.open(buggyAdapter, { - // Drops the required "text" field instead of carrying it forward: the - // compiler refuses it, and validation still holds for code it can't see. - // @ts-expect-error a migration must return the target's required fields - migrations: [migration(NoteV1, NoteV2, () => ({ title: '' }))], + // What derive() returns is known only when it runs, so validation at + // write time is what refuses it. + migrations: [migration(NoteV1, NoteV2, [derive('text', () => 42)])], }); await buggyStack.defineType(NoteV1); await buggyStack.defineType(NoteV2); @@ -2279,19 +2273,17 @@ describe('migrateAll', () => { }); // migrateAll() and commitMigration() share one checked write path: a - // migration function is app code, but so is the app calling + // migration is app code, but so is the app calling // commitMigration(), and neither is entitled to move a DID binding or // slip a reserved key past validation. - test('aborts when a migration function would move a DID binding', async () => { + test('aborts when a migration would move a DID binding', async () => { const EntityV2 = typeHandle({ id: '_entity@2', name: 'Entity', schema: EntityV1.schema, migratesFrom: EntityV1, }); - await reopenWith( - migration(EntityV1, EntityV2, (content) => ({ ...content, did: 'did:key:zHijacked' })), - ); + await reopenWith(migration(EntityV1, EntityV2, [derive('did', () => 'did:key:zHijacked')])); await stack.defineType(EntityV2); const card = await stack.create('_entity@1', { did: 'did:key:zAlice', name: 'Alice' }); @@ -2302,21 +2294,16 @@ describe('migrateAll', () => { }); }); - test('aborts when a migration function emits a reserved content key', async () => { + test('refuses a migration producing a reserved content key', () => { const NoteV3 = typeHandle({ id: NOTE_V3, name: 'Note', schema: NoteV2.schema, migratesFrom: NoteV2, }); - await reopenWith( - noteV1ToV2, - migration(NoteV2, NoteV3, (content) => ({ ...content, ['__proto__']: 'polluted' })), + expect(() => migration(NoteV2, NoteV3, [derive('__proto__', () => 'polluted')])).toThrow( + StackMigrationError, ); - await stack.defineType(NoteV3); - await stack.create(NOTE_V2, { text: 'hi', title: '' }); - - await expect(stack.migrateAll('com.example.test/note')).rejects.toThrow(StackValidationError); }); test('still carries an unchanged DID binding through a migration', async () => { @@ -2326,9 +2313,7 @@ describe('migrateAll', () => { schema: { ...EntityV1.schema, pronouns: { kind: 'string' } }, migratesFrom: EntityV1, }); - await reopenWith( - migration(EntityV1, EntityV2, (content) => ({ ...content, pronouns: 'they/them' })), - ); + await reopenWith(migration(EntityV1, EntityV2, [add('pronouns', 'they/them')])); await stack.defineType(EntityV2); const card = await stack.create('_entity@1', { did: 'did:key:zAlice', name: 'Alice' }); @@ -4032,7 +4017,7 @@ describe('grantType', () => { const bare = await MemoryAdapter.open({ ownerEntityId: 'owner-123', timezone: 'UTC' }); (bare as { capabilities: StackAdapter['capabilities'] }).capabilities = { ...bare.capabilities, - sort: { fields: [], contentField: false }, + sort: { fields: [], contentField: false, contentFieldByType: 0 }, }; const bareStack = await Stack.open(bare); expect(await bareStack.listTypeGrants()).toEqual([]); @@ -5830,11 +5815,13 @@ describe('nested content paths', () => { filter: { content: 'field', contentPresent: false, + contentByType: 0, search: false, }, sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: false, + contentFieldByType: 0, }, limits: { attachmentBytes: null, @@ -5865,8 +5852,12 @@ describe('limits.contentBytes pre-check', () => { const withContentCeiling = async (contentBytes: number): Promise => Object.assign(await MemoryAdapter.open({ ownerEntityId: 'owner-123', timezone: 'UTC' }), { capabilities: { - filter: { content: 'path', contentPresent: true, search: false }, - sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: true }, + filter: { content: 'path', contentPresent: true, contentByType: 64, search: false }, + sort: { + fields: ['createdAt', 'updatedAt', 'version'], + contentField: true, + contentFieldByType: 64, + }, limits: { attachmentBytes: null, contentBytes }, }, }); @@ -5919,8 +5910,12 @@ describe('putAttachment — limits.attachmentBytes pre-check', () => { const withCeiling = async (attachmentBytes: number): Promise => Object.assign(await MemoryAdapter.open({ ownerEntityId: 'owner-123', timezone: 'UTC' }), { capabilities: { - filter: { content: 'none', contentPresent: false, search: false }, - sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: false }, + filter: { content: 'none', contentPresent: false, contentByType: 0, search: false }, + sort: { + fields: ['createdAt', 'updatedAt', 'version'], + contentField: false, + contentFieldByType: 0, + }, limits: { attachmentBytes, contentBytes: null }, }, }); diff --git a/packages/core/tests/type-handle.test.ts b/packages/core/tests/type-handle.test.ts index 99d23180..18870ef9 100644 --- a/packages/core/tests/type-handle.test.ts +++ b/packages/core/tests/type-handle.test.ts @@ -2,7 +2,13 @@ import { describe, test, expect, expectTypeOf } from 'vitest'; import { Stack } from '../src/stack/stack.js'; import { MemoryAdapter } from '../src/testing.js'; import { migration, typeHandle } from '../src/type-handle.js'; -import type { ContentOf, StoredContentOf, UnlistedValue } from '../src/type-handle.js'; +import { add, derive, drop, map, rename } from '../src/migration-steps.js'; +import type { + ContentOf, + ReadonlyTypeSchema, + StoredContentOf, + UnlistedValue, +} from '../src/type-handle.js'; import { StackBadRequestError, StackMigrationError } from '../src/errors.js'; import type { TypeId, TypeSchema } from '../src/types/index.js'; @@ -195,37 +201,81 @@ describe('typeHandle()', () => { }); }); +const handle = ( + version: number, + schema: S, + migratesFrom?: TypeId, +) => + typeHandle({ + id: `com.example/x@${version}`, + name: 'X', + schema, + ...(migratesFrom && { migratesFrom }), + }); + describe('migration()', () => { - test('types the function from both handles', () => { - const m = migration(BookV1, Book, (c) => { - expectTypeOf(c).toEqualTypeOf>(); - return { ...c, status: c.status === 'done' ? 'finished' : c.status }; - }); + test('builds migrate() from its steps, carrying every field no step names', () => { + const m = migration(BookV1, Book, [map('status', { done: 'finished' })]); expect(m.from).toBe(BookV1.id); expect(m.to).toBe(Book.id); + expect(m.steps).toHaveLength(1); expect(m.migrate({ title: 'Dune', status: 'done' })).toEqual({ title: 'Dune', status: 'finished', }); }); - test('refuses at compile time content missing a required field', () => { - // @ts-expect-error `status` is required in the target - migration(BookV1, Book, (c) => ({ title: c.title })); - }); - - test('carries a value the source handle does not list through', () => { - const m = migration(BookV1, Book, (c) => c); + test('carries a value the source handle does not list through map()', () => { + const m = migration(BookV1, Book, [map('status', { done: 'finished' })]); expect(m.migrate({ title: 'Dune', status: 'paused' })).toEqual({ title: 'Dune', status: 'paused', }); }); - test('refuses at compile time a lookup keyed on only the listed values', () => { - const renamed = { want: 'want', reading: 'reading', done: 'finished' } as const; - // @ts-expect-error a stored status the source does not list has no entry - migration(BookV1, Book, (c) => ({ ...c, status: renamed[c.status] })); + test('applies steps in order, each to what the last left', () => { + const V1 = handle(1, { + author: { kind: 'string', required: true }, + tags: { kind: 'array', items: { kind: 'enum', values: ['a', 'b'] } }, + legacy: { kind: 'boolean' }, + }); + const V2 = handle( + 2, + { + by: { kind: 'string', required: true }, + tags: { kind: 'array', items: { kind: 'enum', values: ['x', 'b'] } }, + priority: { kind: 'enum', values: ['low', 'normal'], required: true }, + initial: { kind: 'string' }, + }, + V1.id, + ); + const m = migration(V1, V2, [ + rename('author', 'by'), + map('tags', { a: 'x' }), + add('priority', 'normal'), + drop('legacy'), + derive('initial', (c) => c.author.slice(0, 1) || undefined), + ]); + expect(m.migrate({ author: 'Ada', tags: ['a', 'b'], legacy: true })).toEqual({ + by: 'Ada', + tags: ['x', 'b'], + priority: 'normal', + initial: 'A', + }); + expect(m.migrate({ author: '' })).toEqual({ by: '', priority: 'normal' }); + }); + + test('types derive() on the source content as stored', () => { + migration(BookV1, Book, [ + derive('status', (c) => { + expectTypeOf(c).toEqualTypeOf>(); + return c.status === 'done' ? 'finished' : c.status; + }), + ]); + // Never run: the check is the compiler's. + void (() => + // @ts-expect-error the source declares no `isbn` + migration(BookV1, Book, [derive('pages', (c) => c.isbn)])); }); test('refuses a target in the same family that does not migrate from the source', () => { @@ -235,18 +285,143 @@ describe('migration()', () => { schema: Book.schema, migratesFrom: Book, }); - expect(() => migration(BookV1, BookV3, () => ({ title: 'x', status: 'want' }))).toThrow( + expect(() => migration(BookV1, BookV3, [])).toThrow( `Cannot migrate "${BookV1.id}" to "${BookV3.id}": "${BookV3.id}" migrates from "${Book.id}".`, ); - expect(() => migration(Book, BookV1, (c) => ({ title: c.title, status: 'want' }))).toThrow( - StackMigrationError, - ); + expect(() => migration(Book, BookV1, [])).toThrow(StackMigrationError); }); test('builds a step into another family without lineage', () => { - const m = migration(Shelf, Book, (c) => ({ title: c.name, status: 'want' })); + const m = migration(Shelf, Book, [rename('name', 'title'), add('status', 'want')]); expect(m.migrate({ name: 'Fiction' })).toEqual({ title: 'Fiction', status: 'want' }); }); + + describe('checks its steps against both schemas', () => { + const V1 = handle(1, { + title: { kind: 'string', required: true }, + status: { kind: 'enum', values: ['open', 'done'] }, + count: { kind: 'number' }, + meta: { kind: 'object', properties: { a: { kind: 'string' } } }, + }); + const at = (schema: ReadonlyTypeSchema) => handle(2, schema, V1.id); + const refused = ( + to: ReturnType, + steps: Parameters>[2], + message: string, + ) => + expect(() => migration(V1, to, steps)).toThrow( + `Cannot migrate "${V1.id}" to "${to.id}": ${message}.`, + ); + const same = at(V1.schema); + + test('a step naming a field that is not there', () => { + refused( + same, + [rename('nope', 'title')], + 'rename("nope", "title"): "nope" is not a field at this point', + ); + refused( + same, + [drop('title'), drop('title')], + 'drop("title"): "title" is not a field at this point', + ); + refused(same, [map('nope', {})], 'map("nope", …): "nope" is not a field at this point'); + }); + + test('a step onto a field that is already there', () => { + refused( + same, + [rename('title', 'count')], + 'rename("title", "count"): "count" is already a field at this point', + ); + refused( + same, + [add('title', 'x')], + 'add("title", …): "title" is already a field at this point', + ); + }); + + test('a field the target does not declare', () => { + const { count: _count, ...rest } = V1.schema; + refused( + at(rest), + [], + '"count" is carried to a Type that does not declare it; drop or rename it', + ); + refused( + at(V1.schema), + [derive('extra', () => 1)], + '"extra" is carried to a Type that does not declare it; drop or rename it', + ); + }); + + test('a required field nothing produces', () => { + refused( + at({ ...V1.schema, due: { kind: 'date', required: true } }), + [], + '"due" is required at the target, and no step produces it', + ); + }); + + test('a carried field the target would refuse', () => { + refused( + at({ ...V1.schema, count: { kind: 'string' } }), + [], + '"count" changes kind from number to string', + ); + refused( + at({ ...V1.schema, count: { kind: 'number', required: true } }), + [], + '"count" is optional at the source and required at the target', + ); + refused( + at({ ...V1.schema, status: { kind: 'enum', values: ['open', 'finished'] } }), + [], + '"status" holds "done", which the target\'s enum does not list', + ); + refused( + at({ ...V1.schema, status: { kind: 'enum', values: ['open'] } }), + [map('status', { done: 'closed' })], + '"status" holds "done" (mapped to "closed"), which the target\'s enum does not list', + ); + refused( + at({ ...V1.schema, meta: { kind: 'object', properties: {} } }), + [], + '"meta.a" is not declared at the target', + ); + }); + + test('a map of a field that holds no strings', () => { + refused( + same, + [map('count', { a: 'b' })], + 'map("count", …): only an enum, string or text field, or an array of one, can be mapped', + ); + }); + + test('an added value the target refuses', () => { + refused( + at({ ...V1.schema, priority: { kind: 'number' } }), + [add('priority', 'high')], + '"priority" is set to a value the target refuses: Expected number, got string', + ); + }); + + test('accepts a widening it can carry', () => { + expect(() => + migration( + V1, + at({ ...V1.schema, status: { kind: 'string' }, title: { kind: 'text', required: true } }), + [], + ), + ).not.toThrow(); + expect(() => + migration(V1, at({ ...V1.schema, status: { kind: 'enum', values: ['open', 'closed'] } }), [ + map('status', { done: 'closed' }), + ]), + ).not.toThrow(); + }); + }); }); describe('defineType()', () => { diff --git a/packages/core/tests/wire-request.test.ts b/packages/core/tests/wire-request.test.ts index 3671da41..983d8498 100644 --- a/packages/core/tests/wire-request.test.ts +++ b/packages/core/tests/wire-request.test.ts @@ -251,6 +251,41 @@ describe('parseQueryBody', () => { expect(() => parseQueryBody({ sort: { contentField: 7 } })).toThrow(StackBadRequestError); }); + test('filter.contentByType travels as one flat arm per typeId', () => { + const arms = { + 'com.example/note@1': { content: { 'a.b': 1 }, contentPresent: ['c'] }, + 'com.example/note@2': {}, + }; + expect(parseQueryBody({ filter: { contentByType: arms } })).toEqual({ + filter: { contentByType: arms }, + }); + for (const contentByType of [ + [], + { 'com.example/note@1': { typeId: 'x' } }, + { 'com.example/note@1': { content: [] } }, + { 'com.example/note@1': { contentPresent: 'c' } }, + ]) { + expect(() => parseQueryBody({ filter: { contentByType } })).toThrow(StackBadRequestError); + } + }); + + test('sort.contentFieldByType travels as one key per typeId, never beside another member', () => { + const contentFieldByType = { + 'com.example/note@1': { field: 'a' }, + 'com.example/note@2': { value: 3 }, + }; + expect(parseQueryBody({ sort: { contentFieldByType, direction: 'desc' } }).sort).toEqual({ + contentFieldByType, + direction: 'desc', + }); + expect(() => parseQueryBody({ sort: { contentField: 'a', contentFieldByType } })).toThrow( + StackBadRequestError, + ); + expect(() => + parseQueryBody({ sort: { contentFieldByType: { 'com.example/note@1': { rank: 1 } } } }), + ).toThrow(/Unknown key/); + }); + test('filter.content carries multi-segment keys through untouched', () => { expect(parseQueryBody({ filter: { content: { 'emails.value': 'a@example.com' } } })).toEqual({ filter: { content: { 'emails.value': 'a@example.com' } }, diff --git a/packages/record-adapter-do-sqlite/tests/record.test.ts b/packages/record-adapter-do-sqlite/tests/record.test.ts index 53a8a6ce..33d56a11 100644 --- a/packages/record-adapter-do-sqlite/tests/record.test.ts +++ b/packages/record-adapter-do-sqlite/tests/record.test.ts @@ -104,11 +104,13 @@ describe('construction', () => { filter: { content: 'path', contentPresent: true, + contentByType: 64, search: true, }, sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: true, + contentFieldByType: 64, }, limits: { attachmentBytes: null, diff --git a/packages/sqlite-shared/src/cursor.ts b/packages/sqlite-shared/src/cursor.ts index f7cebd76..586098a0 100644 --- a/packages/sqlite-shared/src/cursor.ts +++ b/packages/sqlite-shared/src/cursor.ts @@ -8,8 +8,15 @@ */ import { NATIVE_SORT_FIELDS, StackBadRequestError } from '@haverstack/core'; -import { contentSortEntry } from '@haverstack/core/adapter'; -import type { StackRecord, NativeSortField, QuerySort, ScalarFieldKind } from '@haverstack/core'; +import { constantSortEntry, contentSortEntry } from '@haverstack/core/adapter'; +import type { SortEntry } from '@haverstack/core/adapter'; +import type { + ContentSortKey, + StackRecord, + NativeSortField, + QuerySort, + ScalarFieldKind, +} from '@haverstack/core'; /** * Which partition of the ordering a cursor sits in: a native column or a @@ -82,6 +89,14 @@ export const decodeCursor = (cursor: string): DecodedCursor => { throw malformed(); }; +/** + * The name a cursor carries for a `contentFieldByType` sort: the map + * itself, keys in order, so a cursor resumes only the map it was minted + * under. + */ +export const byTypeDescriptor = (byType: Record): string => + JSON.stringify(Object.entries(byType).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); + export const getSortField = (query: { sort?: QuerySort }): NativeSortField => query.sort?.field ?? 'createdAt'; @@ -100,6 +115,17 @@ export const makeCursor = ( sort: QuerySort | undefined, kindOf?: (record: StackRecord, field: string) => ScalarFieldKind | undefined, ): string => { + const byType = sort?.contentFieldByType; + if (byType) { + const field = byTypeDescriptor(byType); + const key = Object.hasOwn(byType, record.typeId) ? byType[record.typeId] : undefined; + const kind = key?.field === undefined ? undefined : kindOf?.(record, key.field); + const entry = + key?.field !== undefined + ? kind && contentSortEntry(kind, record.content[key.field]) + : key && constantSortEntry(key.value ?? null); + return encodeEntry(field, entry || null, record.id); + } const contentField = sort?.contentField; if (contentField === undefined) { const field = getSortField({ sort }); @@ -114,8 +140,12 @@ export const makeCursor = ( const kind = kindOf?.(record, contentField); const entry = kind ? contentSortEntry(kind, record.content[contentField]) : null; - if (!entry) return encodeCursor({ kind: 'absent', field: contentField, id: record.id }); + return encodeEntry(contentField, entry, record.id); +}; + +const encodeEntry = (field: string, entry: SortEntry | null, id: string): string => { + if (!entry) return encodeCursor({ kind: 'absent', field, id }); return entry.kind === 'num' - ? encodeCursor({ kind: 'num', field: contentField, value: entry.num, id: record.id }) - : encodeCursor({ kind: 'text', field: contentField, value: entry.text, id: record.id }); + ? encodeCursor({ kind: 'num', field, value: entry.num, id }) + : encodeCursor({ kind: 'text', field, value: entry.text, id }); }; diff --git a/packages/sqlite-shared/src/query.ts b/packages/sqlite-shared/src/query.ts index 0ebeb7dd..8838048e 100644 --- a/packages/sqlite-shared/src/query.ts +++ b/packages/sqlite-shared/src/query.ts @@ -13,9 +13,22 @@ * fact, which costs a full pass over the table for every page. */ -import { StackBadRequestError, type NativeSortField, type StackQuery } from '@haverstack/core'; -import { contentSortKey, parseContentFilterKey } from '@haverstack/core/adapter'; -import { decodeCursor, getSortColumn, getSortField, type DecodedCursor } from './cursor.js'; +import { + StackBadRequestError, + type ContentArm, + type ContentSortKey, + type NativeSortField, + type StackQuery, +} from '@haverstack/core'; +import { constantSortEntry, contentSortKey, parseContentFilterKey } from '@haverstack/core/adapter'; +import type { SortEntry } from '@haverstack/core/adapter'; +import { + byTypeDescriptor, + decodeCursor, + getSortColumn, + getSortField, + type DecodedCursor, +} from './cursor.js'; import { sanitizeFts5Query } from './fts5.js'; /** One statement plus the parameters it binds, in textual order. */ @@ -136,6 +149,54 @@ const contentCursor = (cursor: DecodedCursor, field: string) => { return cursor; }; +/** + * A `content` and `contentPresent` predicate as conditions on `r`, binding + * in textual order. A dot-separated path is matched element-wise through + * arrays, and a `null` value means "no value at the path, or a value that + * is null" (docs/spec/data-model.md § Filter): the second arm below, "some + * value there is null" OR "nothing is there at all". + */ +const contentConditions = ( + predicate: ContentArm, + params: unknown[], + nextAlias: () => string, +): string[] => { + const conditions: string[] = []; + for (const [key, value] of Object.entries(predicate.content ?? {})) { + const segments = parseContentFilterKey(key); + // undefined is the same absence null names — JSON drops it on the + // way over the wire, so treating it as a value would make an + // in-process query and its wire equivalent disagree. + if (value === null || value === undefined) { + const isNull = contentPathExists(segments, (a) => `${a}.value IS NULL`, params, nextAlias); + const anyValue = contentPathExists( + segments, + (a) => `${a}.value IS NOT NULL`, + params, + nextAlias, + ); + conditions.push(`(${isNull} OR NOT ${anyValue})`); + } else { + const match = contentPathExists(segments, equalsLeaf, params, nextAlias); + params.push(value); + conditions.push(match); + } + } + // Presence — the question an exact-match value cannot ask: a path holds + // a value when at least one non-null value is reachable at it. + for (const key of predicate.contentPresent ?? []) { + conditions.push( + contentPathExists( + parseContentFilterKey(key), + (a) => `${a}.value IS NOT NULL`, + params, + nextAlias, + ), + ); + } + return conditions; +}; + /** * Every condition on the records row itself — everything a query asks * except where in the ordering to resume, which is per-partition and gets @@ -145,6 +206,8 @@ const recordConditions = (query: StackQuery): { conditions: string[]; params: un const conditions: string[] = ["r.id != '_config'"]; const params: unknown[] = []; const f = query.filter ?? {}; + let aliasSeq = 0; + const nextContentAlias = () => `cp${aliasSeq++}`; /** * A filter that accepts one value or a list of them, as the `IN` that @@ -279,52 +342,18 @@ const recordConditions = (query: StackQuery): { conditions: string[]; params: un conditions.push(hasAssociation('relationship', clauses)); } - // Content field filters — a dot-separated path, matched element-wise - // through arrays. A `null` value means "no value at the path, or a value - // that is null" (docs/spec/data-model.md § Filter), which is the second - // arm below: "some value there is null" OR "nothing is there at all". - if (f.content) { - let aliasSeq = 0; - const nextAlias = () => `cp${aliasSeq++}`; - for (const [key, value] of Object.entries(f.content)) { - const segments = parseContentFilterKey(key); - // undefined is the same absence null names — JSON drops it on the - // way over the wire, so treating it as a value would make an - // in-process query and its wire equivalent disagree. - if (value === null || value === undefined) { - const isNull = contentPathExists(segments, (a) => `${a}.value IS NULL`, params, nextAlias); - const anyValue = contentPathExists( - segments, - (a) => `${a}.value IS NOT NULL`, - params, - nextAlias, - ); - conditions.push(`(${isNull} OR NOT ${anyValue})`); - } else { - const match = contentPathExists(segments, equalsLeaf, params, nextAlias); - params.push(value); - conditions.push(match); - } - } + if (f.content || f.contentPresent?.length) { + conditions.push(...contentConditions(f, params, nextContentAlias)); } - // Presence — the question an exact-match value cannot ask. Element-wise - // like the content filter above: a path holds a value when at least one - // non-null value is reachable at it. See docs/spec/data-model.md - // § Filter. - if (f.contentPresent?.length) { - let aliasSeq = 0; - const nextAlias = () => `pp${aliasSeq++}`; - for (const key of f.contentPresent) { - conditions.push( - contentPathExists( - parseContentFilterKey(key), - (a) => `${a}.value IS NOT NULL`, - params, - nextAlias, - ), - ); - } + // One arm per Type, each pinned to its type_id; types partition records, + // so no record meets two arms (docs/spec/data-model.md § Content by type). + if (f.contentByType) { + const arms = Object.entries(f.contentByType).map(([typeId, arm]) => { + params.push(typeId); + return ['r.type_id = ?', ...contentConditions(arm, params, nextContentAlias)].join(' AND '); + }); + conditions.push(`(${arms.map((arm) => `(${arm})`).join(' OR ')})`); } if (f.search) { @@ -393,6 +422,9 @@ export const buildQueryPlan = (query: StackQuery): QueryStatement[] => { const dir = sqlDirection(query); const op = dir === 'ASC' ? '>' : '<'; const decoded = query.cursor ? decodeCursor(query.cursor) : null; + if (query.sort?.contentFieldByType) { + return byTypePlan(query, query.sort.contentFieldByType, dir, op, decoded); + } const contentField = query.sort?.contentField; if (contentField === undefined) { @@ -453,3 +485,98 @@ export const buildQueryPlan = (query: StackQuery): QueryStatement[] => { return pages; }; + +/** + * A sort naming its value per Type: the same two orderings as a content + * field sort, read through a derived table in place of the index. Its rows + * are the index rows of each Type's own field, and one row per record of a + * Type ordered by a constant, carrying the constant in the index's columns + * so one cursor condition and one ORDER BY serve both. See + * docs/spec/data-model.md § Sorting by a content field. + */ +const byTypePlan = ( + query: StackQuery, + byType: Record, + dir: 'ASC' | 'DESC', + op: '>' | '<', + decoded: DecodedCursor | null, +): QueryStatement[] => { + const cursor = decoded ? contentCursor(decoded, byTypeDescriptor(byType)) : null; + const fields: [string, string][] = []; + const constants: [string, SortEntry][] = []; + for (const [typeId, key] of Object.entries(byType)) { + if (key.field !== undefined) { + fields.push([typeId, key.field]); + continue; + } + const entry = constantSortEntry(key.value ?? null); + if (entry) constants.push([typeId, entry]); + } + const pages: QueryStatement[] = []; + const fieldOf = (alias: string, record: string) => + fields.map(() => `(${record}.type_id = ? AND ${alias}.field = ?)`).join(' OR '); + + if (cursor?.kind !== 'absent' && fields.length + constants.length > 0) { + const sources: string[] = []; + const sourceParams: unknown[] = []; + if (fields.length > 0) { + sources.push( + 'SELECT c.record_id AS record_id, c.value_rank AS value_rank, c.num_value AS num_value, ' + + 'c.text_key AS text_key, c.text_value AS text_value ' + + `FROM content_index c JOIN records t ON t.id = c.record_id WHERE ${fieldOf('c', 't')}`, + ); + for (const [typeId, field] of fields) sourceParams.push(typeId, field); + } + for (const [typeId, entry] of constants) { + sources.push( + 'SELECT t.id AS record_id, ? AS value_rank, ? AS num_value, ? AS text_key, ' + + '? AS text_value FROM records t WHERE t.type_id = ?', + ); + sourceParams.push( + ...(entry.kind === 'num' ? [0, entry.num, null, null] : [1, null, entry.key, entry.text]), + typeId, + ); + } + const { conditions, params } = recordConditions(query); + if (cursor) conditions.push(presentCursorCondition(cursor, op, params)); + const from = + `(${sources.join(' UNION ALL ')}) ${SORT_ALIAS} ` + + `JOIN records r ON r.id = ${SORT_ALIAS}.record_id`; + const order = + `${SORT_ALIAS}.value_rank ${dir}, ${SORT_ALIAS}.num_value ${dir}, ` + + `${SORT_ALIAS}.text_key ${dir}, ${SORT_ALIAS}.text_value ${dir}, ` + + `${SORT_ALIAS}.record_id ${dir}`; + pages.push( + statement( + 'r.*', + from, + conditions, + [...sourceParams, ...params], + ` ORDER BY ${order} LIMIT ?`, + ), + ); + } + + // Records holding nothing to order by: a Type's field with no value, a + // `null` constant, or a Type the map does not name. + { + const { conditions, params } = recordConditions(query); + if (fields.length > 0) { + conditions.push( + `NOT EXISTS (SELECT 1 FROM content_index ${SORT_ALIAS} ` + + `WHERE ${SORT_ALIAS}.record_id = r.id AND (${fieldOf(SORT_ALIAS, 'r')}))`, + ); + for (const [typeId, field] of fields) params.push(typeId, field); + } + if (constants.length > 0) { + conditions.push(`r.type_id NOT IN (${constants.map(() => '?').join(',')})`); + params.push(...constants.map(([typeId]) => typeId)); + } + if (cursor?.kind === 'absent') { + conditions.push(`r.id ${op} ?`); + params.push(cursor.id); + } + pages.push(statement('r.*', 'records r', conditions, params, ` ORDER BY r.id ${dir} LIMIT ?`)); + } + return pages; +}; diff --git a/packages/sqlite-shared/src/record-adapter.ts b/packages/sqlite-shared/src/record-adapter.ts index 8c337083..9c069b02 100644 --- a/packages/sqlite-shared/src/record-adapter.ts +++ b/packages/sqlite-shared/src/record-adapter.ts @@ -50,11 +50,15 @@ export const SQLITE_RECORD_CAPABILITIES: StackCapabilities = { filter: { content: 'path', contentPresent: true, + // Inside SQLite's limits on expression depth and compound selects; see + // docs/spec/adapters.md § Adapter capabilities. + contentByType: 64, search: true, }, sort: { fields: ['createdAt', 'updatedAt', 'version'], contentField: true, + contentFieldByType: 64, }, limits: { attachmentBytes: null, diff --git a/packages/wire-types/src/discovery.ts b/packages/wire-types/src/discovery.ts index 311a058e..d1fc4822 100644 --- a/packages/wire-types/src/discovery.ts +++ b/packages/wire-types/src/discovery.ts @@ -39,8 +39,8 @@ export type DiscoveryResponse = { * checks rather than casts. See docs/spec/wire-format.md § Discovery. */ export type DiscoveryCapabilities = { - filter?: { content?: string; contentPresent?: boolean; search?: boolean }; - sort?: { fields?: string[]; contentField?: boolean }; + filter?: { content?: string; contentPresent?: boolean; contentByType?: number; search?: boolean }; + sort?: { fields?: string[]; contentField?: boolean; contentFieldByType?: number }; limits?: { attachmentBytes?: number | null; contentBytes?: number | null }; }; @@ -60,16 +60,23 @@ export function normalizeCapabilities( const filter = capabilities?.filter; const sort = capabilities?.sort; const limits = capabilities?.limits; - const reach = filter?.content; + const reach = CONTENT_FILTER_REACHES.has(filter?.content as string) + ? (filter?.content as ContentFilterReach) + : 'none'; + // The per-Type shapes travel in the POST /records/query body, which a + // server reaching no content does not serve. + const cap = (value: unknown): number => (reach === 'none' ? 0 : capOrZero(value)); return { filter: { - content: CONTENT_FILTER_REACHES.has(reach as string) ? (reach as ContentFilterReach) : 'none', + content: reach, contentPresent: filter?.contentPresent === true, + contentByType: cap(filter?.contentByType), search: filter?.search === true, }, sort: { fields: (sort?.fields ?? []).filter((f): f is NativeSortField => SORT_FIELDS.has(f)), contentField: sort?.contentField === true, + contentFieldByType: cap(sort?.contentFieldByType), }, limits: { attachmentBytes: finiteOrNull(limits?.attachmentBytes), @@ -78,6 +85,10 @@ export function normalizeCapabilities( }; } +/** A cap on keys, or 0 — declined — for anything but a non-negative integer. */ +const capOrZero = (value: unknown): number => + typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : 0; + /** A ceiling this client can compare against, or null for "cannot pre-check". */ const finiteOrNull = (value: unknown): number | null => typeof value === 'number' && Number.isFinite(value) ? value : null; diff --git a/packages/wire-types/tests/changes.test.ts b/packages/wire-types/tests/changes.test.ts index 70745718..7c1e5c0c 100644 --- a/packages/wire-types/tests/changes.test.ts +++ b/packages/wire-types/tests/changes.test.ts @@ -42,11 +42,13 @@ const discovery = (overrides: Partial = {}): DiscoveryRespons filter: { content: 'path', contentPresent: true, + contentByType: 64, search: true, }, sort: { fields: ['createdAt'], contentField: true, + contentFieldByType: 64, }, limits: { attachmentBytes: null, diff --git a/packages/wire-types/tests/discovery.test.ts b/packages/wire-types/tests/discovery.test.ts index eb7fc374..54583d2d 100644 --- a/packages/wire-types/tests/discovery.test.ts +++ b/packages/wire-types/tests/discovery.test.ts @@ -53,8 +53,8 @@ describe('isProtocolCompatible', () => { // ------------------------------------------------------- const NONE: StackCapabilities = { - filter: { content: 'none', contentPresent: false, search: false }, - sort: { fields: [], contentField: false }, + filter: { content: 'none', contentPresent: false, contentByType: 0, search: false }, + sort: { fields: [], contentField: false, contentFieldByType: 0 }, limits: { attachmentBytes: null, contentBytes: null }, }; @@ -62,13 +62,13 @@ describe('normalizeCapabilities', () => { it('reads what a server declares', () => { expect( normalizeCapabilities({ - filter: { content: 'path', contentPresent: true, search: true }, - sort: { fields: ['createdAt', 'version'], contentField: true }, + filter: { content: 'path', contentPresent: true, contentByType: 64, search: true }, + sort: { fields: ['createdAt', 'version'], contentField: true, contentFieldByType: 64 }, limits: { attachmentBytes: 52428800, contentBytes: 1048576 }, }), ).toEqual({ - filter: { content: 'path', contentPresent: true, search: true }, - sort: { fields: ['createdAt', 'version'], contentField: true }, + filter: { content: 'path', contentPresent: true, contentByType: 64, search: true }, + sort: { fields: ['createdAt', 'version'], contentField: true, contentFieldByType: 64 }, limits: { attachmentBytes: 52428800, contentBytes: 1048576 }, }); }); @@ -82,6 +82,26 @@ describe('normalizeCapabilities', () => { ).toEqual([...NATIVE_SORT_FIELDS]); }); + it('reads a per-Type cap that is not a positive integer as 0', () => { + for (const cap of [-1, 2.5, '16', true, null, Number.POSITIVE_INFINITY]) { + const read = normalizeCapabilities({ + filter: { content: 'path', contentByType: cap as number }, + sort: { contentFieldByType: cap as number }, + }); + expect([read.filter.contentByType, read.sort.contentFieldByType]).toEqual([0, 0]); + } + }); + + // Both travel in the POST /records/query body, which a server reaching no + // content does not serve, so a cap it declares is one it cannot honor. + it('reads a per-Type cap as 0 beside a reach of none', () => { + const read = normalizeCapabilities({ + filter: { content: 'none', contentByType: 16 }, + sort: { contentFieldByType: 16 }, + }); + expect([read.filter.contentByType, read.sort.contentFieldByType]).toEqual([0, 0]); + }); + it('reads an absent capabilities object as declaring nothing', () => { expect(normalizeCapabilities(undefined)).toEqual(NONE); expect(normalizeCapabilities({})).toEqual(NONE); @@ -90,7 +110,7 @@ describe('normalizeCapabilities', () => { it('reads an absent group as declaring nothing, not as inheriting its siblings', () => { expect(normalizeCapabilities({ filter: { content: 'path' } })).toEqual({ ...NONE, - filter: { content: 'path', contentPresent: false, search: false }, + filter: { content: 'path', contentPresent: false, contentByType: 0, search: false }, }); }); From 003c79f5c2759f243058686c604707b145204c17 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 14:02:43 +0000 Subject: [PATCH 2/2] fix(core): carry unnamed fields by name and pin a content query's versions Review on #449: - A field the handle declares and no step produces is read under its own name at an older version, since migrate() carries every field no step names; only a name a step removed is "no value". An older version that gained the field in place was otherwise silently dropped from a filter and sorted last. - A content query's cursor now carries the versions its first page chose, so a version defined mid-pagination cannot change the sort map a cursor resumes, and a later page no longer lists types before querying. - Shorter comments, one withoutContent() helper, validate.ts's isPlainObject() reused, and no "legacy" field in tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LREkawquacdfohXcxKRYWa --- docs/spec/data-model.md | 6 +- packages/core/src/collection.ts | 85 ++++++++++++++++----- packages/core/src/migration-steps.ts | 21 ++++- packages/core/src/query-validation.ts | 9 ++- packages/core/src/type-handle.ts | 9 +-- packages/core/src/validate.ts | 2 +- packages/core/tests/collection.test.ts | 46 ++++++++++- packages/core/tests/migration-steps.test.ts | 29 ++++++- packages/core/tests/type-handle.test.ts | 6 +- packages/sqlite-shared/src/query.ts | 8 +- 10 files changed, 172 insertions(+), 49 deletions(-) diff --git a/docs/spec/data-model.md b/docs/spec/data-model.md index c7d6f50e..3388887d 100644 --- a/docs/spec/data-model.md +++ b/docs/spec/data-model.md @@ -473,9 +473,9 @@ The derived types are a convenience over [runtime validation](#types), which sta | `drop` | nothing to translate: the handle does not declare the field | nothing to translate | | `derive` | narrowed | narrowed | -A field the handle declares optional and no step produces is the constant "no value". A `map` keeps a value it does not list, so more than one older value can arrive as the one a key asks for: `map('status', { done: 'finished' })` carries an unlisted `finished` through as well as remapping `done`. An arm matches one value per path, so it asks for a mapped value first, and each other source becomes a residual the first page probes for, as it probes a narrowed version: a match sets `skipped.older`. One such key per arm is split this way; a second narrows the version. A key no older value maps to drops the older arm, as a constant that does not match does. A key on a field the handle does not declare, which only an untyped caller can write, is narrowed. **A narrowed version is left out of the query**; a dropped arm is left out too, but only because no record there can match. Newer versions stay out of reach either way: the handle cannot represent them. +A field the handle declares optional and no step produces is read under its own name, since a migration carries every field no step names — an older version may have gained it in place — and a name a step removed is the constant "no value". A `map` keeps a value it does not list, so more than one older value can arrive as the one a key asks for: `map('status', { done: 'finished' })` carries an unlisted `finished` through as well as remapping `done`. An arm matches one value per path, so it asks for a mapped value first, and each other source becomes a residual the first page probes for, as it probes a narrowed version: a match sets `skipped.older`. One such key per arm is split this way; a second narrows the version. A key no older value maps to drops the older arm, as a constant that does not match does. A key on a field the handle does not declare, which only an untyped caller can write, is narrowed. **A narrowed version is left out of the query**; a dropped arm is left out too, but only because no record there can match. Newer versions stay out of reach either way: the handle cannot represent them. -Records at an older version arrive migrated, as on an unfiltered page, and a sort orders them as their migrated values would order, since a carried field keeps its kind. When no older version is left in, the query is sent as a plain `content` filter or `contentField` sort on the handle's `typeId`, which every adapter reaching content answers. An adapter declaring [`filter.contentByType`](./adapters.md#adapter-capabilities) `0` — or `sort.contentFieldByType` `0`, for a content sort — gets that query, with every older version narrowed. One declaring a lower cap than the family has versions to query gets the newest versions that fit, and the rest are narrowed. +Records at an older version arrive migrated, as on an unfiltered page, and a sort orders them as their migrated values would order, since a carried field keeps its kind. The first page fixes which versions the query spans, and its cursor carries them: a later page queries the same arms and sort without listing types again, so a version defined mid-pagination cannot change what a cursor resumes. When no older version is left in, the query is sent as a plain `content` filter or `contentField` sort on the handle's `typeId`, which every adapter reaching content answers. An adapter declaring [`filter.contentByType`](./adapters.md#adapter-capabilities) `0` — or `sort.contentFieldByType` `0`, for a content sort — gets that query, with every older version narrowed. One declaring a lower cap than the family has versions to query gets the newest versions that fit, and the rest are narrowed. **`skipped` says which side the query could not reach.** The first page sets `skipped.older` when records at a narrowed older version match the rest of the query, and `skipped.newer` when records at a newer version do: the results may then be missing records the filter was meant to find. A version is reported only when the filter or sort touches a field that does not translate across some step between it and the handle, or when a residual of its arm matches — a dropped arm is not a skip. A later page carries no `skipped`, since the flags depend on the query, not the page. Each is one `limit: 1` read, run beside the first page's own, and a scoped read that comes back empty with a cursor still set counts as a match rather than reading on, so a flag can be set when nothing was skipped but is never clear when something was. Mixed versions are a lasting state, not a backlog, since an app at an older version may keep writing its own, so the app decides what `skipped.older` means to it; `migrateAll()` clears it only until the next such write. `skipped.newer` says the app is behind, which an unfiltered page reports as misfits; a content query's page carries none. A query with no content keys reads the whole family. The untyped `query({ filter: { baseId, content } })` still matches across the family, returning records as stored. @@ -721,7 +721,7 @@ A caller that genuinely needs a count follows `cursor` to exhaustion and counts **Where a query spans Types that declare one field name differently, numbers sort before text**, and `direction` reverses that with everything else. Stating it here rather than leaving it to each engine's collation is what stops two adapters answering one query in two orders. -**`sort.contentFieldByType` names the sort value per Type**, as [`filter.contentByType`](#content-by-type) names a filter per Type, and is how a [collection](#content-queries-across-versions) orders a family across its versions. A Record whose `typeId` is a key orders by that key's `field`, read as its own Type declares it, or by the constant `value`: a number orders as a number, a string as text, and `null` as no value. A Record whose `typeId` is not a key holds no value. The constant arrives already reduced, so an adapter orders it without knowing what kind it stands for — a `date` constant travels as the epoch milliseconds a stored `date` orders by. Each `field` is one segment, as `contentField` is; the map names at least one key and at most the adapter's `sort.contentFieldByType`, and anything else is refused with 400. A cursor resumes only the map it was minted under. +**`sort.contentFieldByType` names the sort value per Type**, as [`filter.contentByType`](#content-by-type) names a filter per Type, and is how a [collection](#content-queries-across-versions) orders a family across its versions. A Record whose `typeId` is a key orders by that key's `field`, read as its own Type declares it, or by the constant `value`: a number orders as a number, a string as text, and `null` as no value. A Record whose `typeId` is not a key holds no value. The constant arrives already reduced, so an adapter orders it without knowing what kind it stands for — a `date` constant travels as the epoch milliseconds a stored `date` orders by. Each `field` is one segment, as `contentField` is; the map names at least one key and at most the adapter's `sort.contentFieldByType`, and anything else is refused with 400. A cursor resumes only the map it was minted under. Ordering across Types cannot walk one field's index in order, so an indexed adapter sorts the candidates on each page; a collection sends the map only when more than one version is in reach. **A content sort orders the stored shape, not the migrated one.** Migrations run in memory after the query returns (see [Type migrations](#type-migrations)), so a Stack mid-migration, sorting on a field a migration rewrites, gets an order derived from what is on disk. The distinction cannot arise for the native three. diff --git a/packages/core/src/collection.ts b/packages/core/src/collection.ts index 266c4874..a6ef5bb5 100644 --- a/packages/core/src/collection.ts +++ b/packages/core/src/collection.ts @@ -107,11 +107,9 @@ export type CollectionPage = { misfits: Misfit[]; cursor: string | null; /** - * A content filter or sort reaches the older versions it translates to, - * and no newer one. Each flag is set when records at a version it could - * not reach match, or may match, the rest of the query: the results may - * be missing records the filter was meant to find. On the first page - * only, since it depends on the query rather than the page. See + * Set when records at a version a content filter or sort could not reach + * match, or may match, the rest of the query. On the first page only, + * since it depends on the query rather than the page. See * docs/spec/data-model.md § Content queries across versions. */ skipped?: { older: boolean; newer: boolean }; @@ -297,6 +295,48 @@ const contentPredicate = (filter: RecordFilter): ContentArm | undefined => { return { ...(content && { content }), ...(present && { contentPresent: present }) }; }; +/** `filter` without its content predicate, for a query that asks about the rest. */ +const withoutContent = (filter: RecordFilter): RecordFilter => { + const { content: _content, contentPresent: _present, ...rest } = filter; + return rest; +}; + +/** + * A content query's cursor: the adapter's, with the versions the first page + * queried, so a version defined mid-pagination cannot change the arms or + * the sort a cursor resumes, and a later page lists no types. + */ +const encodeContentCursor = (versions: TypeId[], cursor: string): string => { + const bytes = new TextEncoder().encode(JSON.stringify({ v: versions, c: cursor })); + let binary = ''; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +}; + +const decodeContentCursor = ( + raw: string, + handle: TypeHandle, +): { versions: TypeId[]; cursor: string } => { + const malformed = () => new StackBadRequestError(`Invalid cursor: malformed "${raw}"`); + let parsed: unknown; + try { + const binary = atob(raw); + const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0)); + parsed = JSON.parse(new TextDecoder().decode(bytes)); + } catch { + throw malformed(); + } + const { v, c } = (parsed ?? {}) as { v?: unknown; c?: unknown }; + if ( + typeof c !== 'string' || + !Array.isArray(v) || + !v.every((id) => typeof id === 'string' && parseTypeId(id)?.baseId === handle.baseId) + ) { + throw malformed(); + } + return { versions: v as TypeId[], cursor: c }; +}; + /** * The collection over `handle`, shared by `Stack.collection()` and * `ScopedStack.collection()`. `registry` is the opening `Stack`'s, which @@ -403,8 +443,7 @@ export const createCollection = ( /** Whether a record stored at one of `typeIds` matches `filter` without its content keys. */ const anyAt = async (typeIds: TypeId[], filter: RecordFilter): Promise => { if (typeIds.length === 0) return false; - const { content: _content, contentPresent: _present, ...rest } = filter; - return anyMatch({ ...rest, typeId: typeIds }); + return anyMatch({ ...withoutContent(filter), typeId: typeIds }); }; /** @@ -412,7 +451,7 @@ export const createCollection = ( * residual is that arm with another source value in place, at its typeId. */ const anyResidual = async (residuals: RecordFilter[], filter: RecordFilter): Promise => { - const { content: _content, contentPresent: _present, ...rest } = filter; + const rest = withoutContent(filter); const found = await Promise.all( residuals.map((residual) => anyMatch({ ...rest, ...residual })), ); @@ -420,18 +459,18 @@ export const createCollection = ( }; /** - * The query a content filter or sort becomes: one arm per version the - * migrations carry it to, the older versions left out of it, and the - * residual predicates an arm could not ask for alongside its own. The - * first page probes the last two for `skipped.older`. Newer versions are - * always out of reach. See docs/spec/data-model.md § Content queries - * across versions. + * A content query as one arm per version it translates to, plus what the + * first page probes for `skipped`. `pinned` is the versions a first page + * chose, which a later page reuses. See docs/spec/data-model.md § Content + * queries across versions. */ const planContentQuery = async ( filter: RecordFilter, sort: StackQuery['sort'], + pinned?: TypeId[], ): Promise<{ query: Pick; + versions: TypeId[]; narrowed: TypeId[]; newer: TypeId[]; residuals: RecordFilter[]; @@ -454,7 +493,11 @@ export const createCollection = ( return def && def.kind !== 'array' && def.kind !== 'object' ? def.kind : undefined; })(); - const family = (await listTypes()) + const family = ( + pinned + ? pinned.map((id) => ({ id, baseId: handle.baseId, version: versionOf(id) })) + : await listTypes() + ) .filter((type) => type.baseId === handle.baseId && type.version !== handleVersion) .sort((a, b) => b.version - a.version); const arms: Record = { [handle.id]: predicate ?? {} }; @@ -495,11 +538,12 @@ export const createCollection = ( for (const arm of translated.residuals) residuals.push({ ...arm, typeId: type.id }); } - const { content: _content, contentPresent: _present, ...rest } = filter; + const rest = withoutContent(filter); const typeIds = Object.keys(arms); if (typeIds.length === 1) { return { query: { filter: { ...filter, typeId: handle.id }, sort }, + versions: typeIds, narrowed, newer, residuals, @@ -512,6 +556,7 @@ export const createCollection = ( ? { contentFieldByType: keys, ...(sort?.direction && { direction: sort.direction }) } : sort, }, + versions: typeIds, narrowed, newer, residuals, @@ -568,14 +613,16 @@ export const createCollection = ( })); if (firstPage) skipped = { older: false, newer: false }; } else { - const plan = await planContentQuery(unversioned, rest.sort); + const resumed = firstPage ? undefined : decodeContentCursor(rest.cursor!, handle); + const plan = await planContentQuery(unversioned, rest.sort, resumed?.versions); const [page, narrowed, residual, newer] = await Promise.all([ - client.query({ ...rest, ...plan.query }), + client.query({ ...rest, cursor: resumed?.cursor, ...plan.query }), firstPage && anyAt(plan.narrowed, unversioned), firstPage && anyResidual(plan.residuals, unversioned), firstPage && anyAt(plan.newer, unversioned), ]); - ({ records, cursor } = page); + records = page.records; + cursor = page.cursor && encodeContentCursor(plan.versions, page.cursor); if (firstPage) skipped = { older: narrowed || residual, newer }; } const page: CollectionPage = { diff --git a/packages/core/src/migration-steps.ts b/packages/core/src/migration-steps.ts index 80e9facf..ab3e492f 100644 --- a/packages/core/src/migration-steps.ts +++ b/packages/core/src/migration-steps.ts @@ -144,7 +144,10 @@ export type FieldOrigin = | { readonly kind: 'constant'; readonly value: unknown } | { readonly kind: 'derived' }; -/** Every top-level field the target declares, by its origin. */ +/** + * Every top-level field the target declares, by its origin, and every name + * the steps removed, as no value. + */ export type FieldOrigins = ReadonlyMap; /** `outer` after `inner`, as one table over both domains. */ @@ -174,7 +177,8 @@ export const composeOrigins = (lower: FieldOrigins, upper: FieldOrigins): FieldO composed.set(field, origin); continue; } - const inner = lower.get(origin.field) ?? { kind: 'derived' }; + // A name the lower step neither declares nor removed passes through it. + const inner = lower.get(origin.field) ?? { kind: 'field', field: origin.field }; composed.set( field, inner.kind === 'field' @@ -275,6 +279,7 @@ export const fieldOrigins = ( const current = new Map( Object.keys(from).map((field) => [field, { kind: 'field', field }]), ); + const removed = new Set(); for (const step of steps) { const at = describe(step); const named = step.op === 'rename' ? step.from : step.field; @@ -286,6 +291,7 @@ export const fieldOrigins = ( if (current.has(step.to)) return `${at}: "${step.to}" is already a field at this point`; current.set(step.to, current.get(step.from)!); current.delete(step.from); + removed.add(step.from); break; case 'map': { const origin = current.get(step.field)!; @@ -310,6 +316,7 @@ export const fieldOrigins = ( break; case 'drop': current.delete(step.field); + removed.add(step.field); break; case 'derive': current.set(step.field, { kind: 'derived' }); @@ -326,7 +333,12 @@ export const fieldOrigins = ( const origin = current.get(field); if (!origin) { if (def.required) return `"${field}" is required at the target, and no step produces it`; - current.set(field, { kind: 'constant', value: undefined }); + // migrate() carries every field no step names, so one an older + // version gained in place arrives under its own name. + current.set( + field, + removed.has(field) ? { kind: 'constant', value: undefined } : { kind: 'field', field }, + ); continue; } if (origin.kind === 'field') { @@ -344,6 +356,9 @@ export const fieldOrigins = ( return `"${field}" is set to a value the target refuses: ${errors[0].message}`; } } + for (const field of removed) { + if (!current.has(field)) current.set(field, { kind: 'constant', value: undefined }); + } return current; }; diff --git a/packages/core/src/query-validation.ts b/packages/core/src/query-validation.ts index 8a526ced..1fe0143f 100644 --- a/packages/core/src/query-validation.ts +++ b/packages/core/src/query-validation.ts @@ -19,7 +19,11 @@ import { ARGUMENTS_INVALID, StackBadRequestError, StackValidationError } from './errors.js'; import { familyIdProblem, parseTypeId } from './schema.js'; import { targetErrors } from './associations/validation.js'; -import { CONTENT_SEGMENT_METACHARACTERS, SEGMENT_METACHARACTER_RE } from './validate.js'; +import { + CONTENT_SEGMENT_METACHARACTERS, + SEGMENT_METACHARACTER_RE, + isPlainObject, +} from './validate.js'; import { NATIVE_SORT_FIELDS } from './types/index.js'; import type { JournalQuery, @@ -95,9 +99,6 @@ export function assertQueryCapabilities( } } -const isPlainObject = (value: unknown): value is Record => - typeof value === 'object' && value !== null && !Array.isArray(value); - /** * Hold `filter.contentByType` to a split along Types: at least one arm, * each pinned to a TypeId and flat, and never beside a top-level content diff --git a/packages/core/src/type-handle.ts b/packages/core/src/type-handle.ts index 257f7a09..69757d8f 100644 --- a/packages/core/src/type-handle.ts +++ b/packages/core/src/type-handle.ts @@ -195,12 +195,9 @@ export type Migration, To = Record( from: TypeHandle, diff --git a/packages/core/src/validate.ts b/packages/core/src/validate.ts index 87eebd32..3f260f04 100644 --- a/packages/core/src/validate.ts +++ b/packages/core/src/validate.ts @@ -341,7 +341,7 @@ const SCALAR_KINDS: Record = { enum: true, }; -const isPlainObject = (value: unknown): value is Record => +export const isPlainObject = (value: unknown): value is Record => typeof value === 'object' && value !== null && !Array.isArray(value); const validateEnumValuesShape = (list: unknown, path: string, errors: ValidationError[]): void => { diff --git a/packages/core/tests/collection.test.ts b/packages/core/tests/collection.test.ts index 0085c9ff..f1b37124 100644 --- a/packages/core/tests/collection.test.ts +++ b/packages/core/tests/collection.test.ts @@ -303,7 +303,21 @@ describe.each([ expect(page.skipped).toEqual({ older: false, newer: false }); }); - test('a field no step produces is no value at an older version', async () => { + test('a field an older version gained in place is found under its own name', async () => { + await stack.defineType({ + ...BookV1, + schema: { ...BookV1.schema, pages: { kind: 'number' } }, + }); + const old = await stack.create(BookV1.id, { title: 'Emma', status: 'want', pages: 474 }); + await createV1('Dune'); + const page = await books.query({ filter: { content: { pages: 474 } } }); + expect(page.records.map((r) => r.id)).toEqual([old.id]); + expect(page.skipped).toEqual({ older: false, newer: false }); + const sorted = await books.query({ sort: { contentField: 'pages' } }); + expect(sorted.records[0].id).toBe(old.id); + }); + + test('a field no step produces holds what an older record stores there', async () => { const book = await books.create({ title: 'Dune', status: 'want', pages: 412 }); const old = await createV1(); const absent = await books.query({ filter: { content: { pages: null } } }); @@ -638,7 +652,7 @@ const TaskV1 = typeHandle({ schema: { author: { kind: 'string', required: true }, status: { kind: 'enum', values: ['open', 'done'], required: true }, - legacy: { kind: 'boolean' }, + archived: { kind: 'boolean' }, }, }); @@ -671,7 +685,7 @@ const TASK_MIGRATIONS = [ rename('author', 'by'), map('status', { done: 'finished' }), add('priority', 'normal'), - drop('legacy'), + drop('archived'), derive('slug', (c) => c.author.toLowerCase()), ]), migration(TaskV2, Task, [rename('by', 'owner')]), @@ -768,6 +782,32 @@ describe.each([ ); }); + test('a later page reuses the versions the first page chose, without listing types', async () => { + adapter = await MemoryAdapter.open({ ownerEntityId: OWNER, timezone: 'UTC' }); + stack = await Stack.open(adapter, { migrations: TASK_MIGRATIONS }); + await stack.defineType(TaskV2); + await stack.defineType(Task); + for (const owner of ['Ada', 'Bea']) { + await stack.create(TaskV2.id, { by: owner, status: 'open', priority: 'low' }); + await stack.create(Task.id, { owner, status: 'open', priority: 'low' }); + } + const query = { sort: { contentField: 'owner' as const }, limit: 2 }; + const first = await tasks().query(query); + + await stack.defineType(TaskV1); + await stack.create(TaskV1.id, { author: 'Ada', status: 'open' }); + const listTypes = vi.spyOn(stack, 'listTypes'); + const second = await tasks().query({ ...query, cursor: first.cursor! }); + + expect([...first.records, ...second.records].map((r) => r.content.owner)).toEqual([ + 'Ada', + 'Ada', + 'Bea', + 'Bea', + ]); + expect(listTypes).not.toHaveBeenCalled(); + }); + test('an adapter that declines the shape gets the handle-only query', async () => { await open(0, 0); const [, , v3] = await seed('Ada'); diff --git a/packages/core/tests/migration-steps.test.ts b/packages/core/tests/migration-steps.test.ts index 5d0be638..245a5841 100644 --- a/packages/core/tests/migration-steps.test.ts +++ b/packages/core/tests/migration-steps.test.ts @@ -1,5 +1,13 @@ import { describe, test, expect } from 'vitest'; -import { add, composeOrigins, derive, fieldOrigins, map, rename } from '../src/migration-steps.js'; +import { + add, + composeOrigins, + derive, + drop, + fieldOrigins, + map, + rename, +} from '../src/migration-steps.js'; import type { FieldOrigins } from '../src/migration-steps.js'; import type { ReadonlyTypeSchema } from '../src/type-handle.js'; @@ -26,7 +34,8 @@ describe('fieldOrigins()', () => { b: { kind: 'field', field: 'a', values: { x: 'y' } }, c: { kind: 'constant', value: 'k' }, d: { kind: 'derived' }, - e: { kind: 'constant', value: undefined }, + e: { kind: 'field', field: 'e' }, + a: { kind: 'constant', value: undefined }, }); }); @@ -50,6 +59,22 @@ describe('fieldOrigins()', () => { }); describe('composeOrigins()', () => { + test('follows a name no step touches through a step that does not declare it', () => { + const lower = origins([], { a: { kind: 'string' } }, { a: { kind: 'string' } }); + const upper = origins( + [], + { a: { kind: 'string' } }, + { a: { kind: 'string' }, n: { kind: 'string' } }, + ); + expect(composeOrigins(lower, upper).get('n')).toEqual({ kind: 'field', field: 'n' }); + }); + + test('reads a name a lower step removed as no value, though an upper one declares it', () => { + const lower = origins([drop('n')], { n: { kind: 'string' } }, {}); + const upper = origins([], {}, { n: { kind: 'string' } }); + expect(composeOrigins(lower, upper).get('n')).toEqual({ kind: 'constant', value: undefined }); + }); + test('follows a field through a rename at each step, composing its maps', () => { const lower = origins( [rename('a', 'b'), map('b', { x: 'y' })], diff --git a/packages/core/tests/type-handle.test.ts b/packages/core/tests/type-handle.test.ts index 18870ef9..8b10d613 100644 --- a/packages/core/tests/type-handle.test.ts +++ b/packages/core/tests/type-handle.test.ts @@ -237,7 +237,7 @@ describe('migration()', () => { const V1 = handle(1, { author: { kind: 'string', required: true }, tags: { kind: 'array', items: { kind: 'enum', values: ['a', 'b'] } }, - legacy: { kind: 'boolean' }, + archived: { kind: 'boolean' }, }); const V2 = handle( 2, @@ -253,10 +253,10 @@ describe('migration()', () => { rename('author', 'by'), map('tags', { a: 'x' }), add('priority', 'normal'), - drop('legacy'), + drop('archived'), derive('initial', (c) => c.author.slice(0, 1) || undefined), ]); - expect(m.migrate({ author: 'Ada', tags: ['a', 'b'], legacy: true })).toEqual({ + expect(m.migrate({ author: 'Ada', tags: ['a', 'b'], archived: true })).toEqual({ by: 'Ada', tags: ['x', 'b'], priority: 'normal', diff --git a/packages/sqlite-shared/src/query.ts b/packages/sqlite-shared/src/query.ts index 8838048e..682d0b91 100644 --- a/packages/sqlite-shared/src/query.ts +++ b/packages/sqlite-shared/src/query.ts @@ -487,11 +487,9 @@ export const buildQueryPlan = (query: StackQuery): QueryStatement[] => { }; /** - * A sort naming its value per Type: the same two orderings as a content - * field sort, read through a derived table in place of the index. Its rows - * are the index rows of each Type's own field, and one row per record of a - * Type ordered by a constant, carrying the constant in the index's columns - * so one cursor condition and one ORDER BY serve both. See + * A sort naming its value per Type, read through a derived table in place + * of the index: each Type's own index rows, and its constant in the index's + * columns, so one cursor condition and one ORDER BY serve both. See * docs/spec/data-model.md § Sorting by a content field. */ const byTypePlan = (